Initial commit: Daily tech newsletter worker

A Node.js/TypeScript worker that sends daily AI-generated summaries
of tech news from X/Twitter via Nitter RSS feeds.

Features:
- Fetches tweets from 40+ curated tech accounts via Nitter RSS
- Filters and categorizes by topic (AI/ML, SWE, General Tech)
- Generates AI summaries using OpenRouter (Claude/GPT-4)
- Sends professional HTML email via Brevo SMTP
- Runs on cron schedule inside Docker container
- Instance rotation for Nitter reliability
- Graceful degradation if AI fails

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

src/config/accounts.ts

@@ -0,0 +1,60 @@
+import type { TechAccount } from '../types/index.js';
+
+export const TECH_ACCOUNTS: TechAccount[] = [
+  // ===========================================
+  // AI / Machine Learning
+  // ===========================================
+  { username: 'karpathy', displayName: 'Andrej Karpathy', category: 'ai_ml', priority: 'high' },
+  { username: 'ylecun', displayName: 'Yann LeCun', category: 'ai_ml', priority: 'high' },
+  { username: 'AndrewYNg', displayName: 'Andrew Ng', category: 'ai_ml', priority: 'high' },
+  { username: 'sama', displayName: 'Sam Altman', category: 'ai_ml', priority: 'high' },
+  { username: 'demaboris', displayName: 'Demis Hassabis', category: 'ai_ml', priority: 'high' },
+  { username: 'goodaboris', displayName: 'Ian Goodfellow', category: 'ai_ml', priority: 'medium' },
+  { username: 'fchollet', displayName: 'François Chollet', category: 'ai_ml', priority: 'high' },
+  { username: 'EMostaque', displayName: 'Emad Mostaque', category: 'ai_ml', priority: 'medium' },
+  { username: 'JimFan', displayName: 'Jim Fan', category: 'ai_ml', priority: 'high' },
+  { username: 'gaborig', displayName: 'George Hotz', category: 'ai_ml', priority: 'medium' },
+  { username: 'ClaudeMcAI', displayName: 'Claude (Anthropic)', category: 'ai_ml', priority: 'medium' },
+  { username: 'OpenAI', displayName: 'OpenAI', category: 'ai_ml', priority: 'high' },
+  { username: 'AnthropicAI', displayName: 'Anthropic', category: 'ai_ml', priority: 'high' },
+  { username: 'huggingface', displayName: 'Hugging Face', category: 'ai_ml', priority: 'high' },
+
+  // ===========================================
+  // Software Engineering / Dev Tools
+  // ===========================================
+  { username: 'ThePrimeagen', displayName: 'ThePrimeagen', category: 'swe', priority: 'high' },
+  { username: 'kelseyhightower', displayName: 'Kelsey Hightower', category: 'swe', priority: 'high' },
+  { username: 'mitchellh', displayName: 'Mitchell Hashimoto', category: 'swe', priority: 'high' },
+  { username: 'tjholowaychuk', displayName: 'TJ Holowaychuk', category: 'swe', priority: 'medium' },
+  { username: 'addyosmani', displayName: 'Addy Osmani', category: 'swe', priority: 'medium' },
+  { username: 'sarah_edo', displayName: 'Sarah Drasner', category: 'swe', priority: 'medium' },
+  { username: 'dan_abramov', displayName: 'Dan Abramov', category: 'swe', priority: 'high' },
+  { username: 'swyx', displayName: 'swyx', category: 'swe', priority: 'medium' },
+  { username: 'kentcdodds', displayName: 'Kent C. Dodds', category: 'swe', priority: 'medium' },
+  { username: 'taborwang', displayName: 'Tanner Linsley', category: 'swe', priority: 'medium' },
+  { username: 'raaborig', displayName: 'Ryan Dahl', category: 'swe', priority: 'high' },
+  { username: 'vercel', displayName: 'Vercel', category: 'swe', priority: 'medium' },
+  { username: 'github', displayName: 'GitHub', category: 'swe', priority: 'medium' },
+
+  // ===========================================
+  // General Tech / Startups
+  // ===========================================
+  { username: 'levelsio', displayName: 'Pieter Levels', category: 'general_tech', priority: 'high' },
+  { username: 'paulg', displayName: 'Paul Graham', category: 'general_tech', priority: 'high' },
+  { username: 'naval', displayName: 'Naval Ravikant', category: 'general_tech', priority: 'high' },
+  { username: 'elaborig', displayName: 'Elon Musk', category: 'general_tech', priority: 'medium' },
+  { username: 'jason', displayName: 'Jason Calacanis', category: 'general_tech', priority: 'medium' },
+  { username: 'balajis', displayName: 'Balaji Srinivasan', category: 'general_tech', priority: 'medium' },
+  { username: 'pmarca', displayName: 'Marc Andreessen', category: 'general_tech', priority: 'medium' },
+  { username: 'aborig', displayName: 'DHH', category: 'general_tech', priority: 'high' },
+  { username: 'benedictevans', displayName: 'Benedict Evans', category: 'general_tech', priority: 'medium' },
+  { username: 'jason_f', displayName: 'Jason Fried', category: 'general_tech', priority: 'medium' },
+];
+
+export function getAccountsByCategory(category: TechAccount['category']): TechAccount[] {
+  return TECH_ACCOUNTS.filter((account) => account.category === category);
+}
+
+export function getHighPriorityAccounts(): TechAccount[] {
+  return TECH_ACCOUNTS.filter((account) => account.priority === 'high');
+}

src/config/index.ts

@@ -0,0 +1,89 @@
+import { config as dotenvConfig } from 'dotenv';
+import { z } from 'zod';
+import type { AppConfig } from '../types/index.js';
+
+dotenvConfig();
+
+const envSchema = z.object({
+  NODE_ENV: z.enum(['development', 'production', 'test']).default('development'),
+  LOG_LEVEL: z.enum(['debug', 'info', 'warn', 'error']).default('info'),
+
+  NITTER_INSTANCES: z.string().default('https://nitter.poast.org,https://xcancel.com'),
+  RSS_FETCH_TIMEOUT: z.coerce.number().default(30000),
+  RSS_MAX_TWEETS_PER_ACCOUNT: z.coerce.number().default(50),
+
+  OPENROUTER_API_KEY: z.string().min(1, 'OPENROUTER_API_KEY is required'),
+  OPENROUTER_MODEL: z.string().default('anthropic/claude-3-sonnet-20240229'),
+  OPENROUTER_MAX_TOKENS: z.coerce.number().default(2000),
+  OPENROUTER_SITE_URL: z.string().default('https://tech-newsletter.local'),
+  OPENROUTER_SITE_NAME: z.string().default('Tech Newsletter'),
+
+  BREVO_SMTP_HOST: z.string().default('smtp-relay.brevo.com'),
+  BREVO_SMTP_PORT: z.coerce.number().default(587),
+  BREVO_SMTP_USER: z.string().min(1, 'BREVO_SMTP_USER is required'),
+  BREVO_SMTP_KEY: z.string().min(1, 'BREVO_SMTP_KEY is required'),
+  EMAIL_FROM_ADDRESS: z.string().email('Invalid EMAIL_FROM_ADDRESS'),
+  EMAIL_FROM_NAME: z.string().default('Daily Tech Digest'),
+  EMAIL_RECIPIENTS: z.string().min(1, 'EMAIL_RECIPIENTS is required'),
+
+  CRON_SCHEDULE: z.string().default('0 7 * * *'),
+  CRON_TIMEZONE: z.string().default('Europe/Warsaw'),
+
+  ENABLE_AI_SUMMARIES: z.coerce.boolean().default(true),
+  INCLUDE_RETWEETS: z.coerce.boolean().default(false),
+  INCLUDE_REPLIES: z.coerce.boolean().default(false),
+  DRY_RUN: z.coerce.boolean().default(false),
+});
+
+function loadConfig(): AppConfig {
+  const result = envSchema.safeParse(process.env);
+
+  if (!result.success) {
+    console.error('Configuration validation failed:');
+    for (const error of result.error.errors) {
+      console.error(`  - ${error.path.join('.')}: ${error.message}`);
+    }
+    process.exit(1);
+  }
+
+  const env = result.data;
+
+  return {
+    rss: {
+      nitterInstances: env.NITTER_INSTANCES.split(',').map((s) => s.trim()),
+      fetchTimeout: env.RSS_FETCH_TIMEOUT,
+      maxTweetsPerAccount: env.RSS_MAX_TWEETS_PER_ACCOUNT,
+    },
+    ai: {
+      openRouterApiKey: env.OPENROUTER_API_KEY,
+      model: env.OPENROUTER_MODEL,
+      maxTokens: env.OPENROUTER_MAX_TOKENS,
+      siteUrl: env.OPENROUTER_SITE_URL,
+      siteName: env.OPENROUTER_SITE_NAME,
+    },
+    email: {
+      brevoHost: env.BREVO_SMTP_HOST,
+      brevoPort: env.BREVO_SMTP_PORT,
+      brevoUser: env.BREVO_SMTP_USER,
+      brevoApiKey: env.BREVO_SMTP_KEY,
+      fromEmail: env.EMAIL_FROM_ADDRESS,
+      fromName: env.EMAIL_FROM_NAME,
+      recipients: env.EMAIL_RECIPIENTS.split(',').map((s) => s.trim()),
+    },
+    scheduler: {
+      cronExpression: env.CRON_SCHEDULE,
+      timezone: env.CRON_TIMEZONE,
+    },
+    features: {
+      enableAiSummaries: env.ENABLE_AI_SUMMARIES,
+      includeRetweets: env.INCLUDE_RETWEETS,
+      includeReplies: env.INCLUDE_REPLIES,
+      dryRun: env.DRY_RUN,
+    },
+    logging: {
+      level: env.LOG_LEVEL,
+    },
+  };
+}
+
+export const config = loadConfig();

src/config/topics.ts

@@ -0,0 +1,210 @@
+import type { TopicConfig, TopicId } from '../types/index.js';
+
+export const TOPICS: TopicConfig[] = [
+  {
+    id: 'ai_ml',
+    name: 'AI & Machine Learning',
+    keywords: [
+      'gpt',
+      'gpt-4',
+      'gpt-5',
+      'chatgpt',
+      'llm',
+      'large language model',
+      'transformer',
+      'neural network',
+      'deep learning',
+      'machine learning',
+      'ml',
+      'artificial intelligence',
+      'ai',
+      'openai',
+      'anthropic',
+      'claude',
+      'gemini',
+      'mistral',
+      'llama',
+      'fine-tuning',
+      'fine tuning',
+      'rag',
+      'retrieval augmented',
+      'embedding',
+      'vector database',
+      'prompt engineering',
+      'prompt',
+      'diffusion',
+      'stable diffusion',
+      'midjourney',
+      'dall-e',
+      'sora',
+      'multimodal',
+      'vision model',
+      'nlp',
+      'natural language',
+      'rlhf',
+      'reinforcement learning',
+      'model training',
+      'inference',
+      'hugging face',
+      'pytorch',
+      'tensorflow',
+      'jax',
+      'agi',
+      'superintelligence',
+      'ai safety',
+      'alignment',
+    ],
+    icon: '🤖',
+  },
+  {
+    id: 'swe',
+    name: 'Software Engineering',
+    keywords: [
+      'programming',
+      'coding',
+      'software',
+      'developer',
+      'development',
+      'typescript',
+      'javascript',
+      'python',
+      'rust',
+      'go',
+      'golang',
+      'java',
+      'c++',
+      'react',
+      'vue',
+      'angular',
+      'svelte',
+      'node',
+      'nodejs',
+      'deno',
+      'bun',
+      'next.js',
+      'nextjs',
+      'remix',
+      'api',
+      'rest',
+      'graphql',
+      'grpc',
+      'microservices',
+      'kubernetes',
+      'k8s',
+      'docker',
+      'container',
+      'devops',
+      'ci/cd',
+      'git',
+      'github',
+      'gitlab',
+      'open source',
+      'oss',
+      'framework',
+      'library',
+      'package',
+      'npm',
+      'testing',
+      'tdd',
+      'clean code',
+      'architecture',
+      'design pattern',
+      'refactoring',
+      'database',
+      'postgresql',
+      'mysql',
+      'mongodb',
+      'redis',
+      'aws',
+      'azure',
+      'gcp',
+      'cloud',
+      'serverless',
+      'edge',
+      'wasm',
+      'webassembly',
+      'vim',
+      'neovim',
+      'vscode',
+      'ide',
+      'terminal',
+      'cli',
+    ],
+    icon: '💻',
+  },
+  {
+    id: 'general_tech',
+    name: 'Tech & Startups',
+    keywords: [
+      'startup',
+      'founder',
+      'entrepreneur',
+      'indie hacker',
+      'indiehacker',
+      'saas',
+      'product',
+      'launch',
+      'shipped',
+      'mvp',
+      'funding',
+      'vc',
+      'venture capital',
+      'seed',
+      'series a',
+      'ipo',
+      'acquisition',
+      'tech news',
+      'silicon valley',
+      'y combinator',
+      'yc',
+      'product hunt',
+      'hacker news',
+      'tech twitter',
+      'remote work',
+      'wfh',
+      'crypto',
+      'blockchain',
+      'web3',
+      'bitcoin',
+      'ethereum',
+      'nft',
+      'defi',
+      'fintech',
+      'biotech',
+      'climate tech',
+      'hardware',
+      'robotics',
+      'automation',
+      'productivity',
+      'notion',
+      'obsidian',
+      'tech layoffs',
+      'hiring',
+      'tech jobs',
+      'career',
+      'salary',
+      'compensation',
+    ],
+    icon: '🚀',
+  },
+];
+
+export function getTopicById(id: TopicId): TopicConfig | undefined {
+  return TOPICS.find((topic) => topic.id === id);
+}
+
+export function matchTopics(text: string): TopicId[] {
+  const lowerText = text.toLowerCase();
+  const matched: TopicId[] = [];
+
+  for (const topic of TOPICS) {
+    for (const keyword of topic.keywords) {
+      if (lowerText.includes(keyword.toLowerCase())) {
+        matched.push(topic.id);
+        break;
+      }
+    }
+  }
+
+  return matched;
+}

src/core/NewsletterPipeline.ts

@@ -0,0 +1,218 @@
+import { config } from '../config/index.js';
+import { TOPICS } from '../config/topics.js';
+import { logger } from '../utils/logger.js';
+import { NitterRssFetcher } from '../services/rss/NitterRssFetcher.js';
+import { TweetProcessor } from './TweetProcessor.js';
+import { SummaryGenerator } from '../services/ai/SummaryGenerator.js';
+import { EmailService } from '../services/email/EmailService.js';
+import type {
+  Newsletter,
+  TopicSummary,
+  PipelineResult,
+  PipelineError,
+  TopicId,
+} from '../types/index.js';
+
+export class NewsletterPipeline {
+  private rssFetcher: NitterRssFetcher;
+  private tweetProcessor: TweetProcessor;
+  private summaryGenerator: SummaryGenerator;
+  private emailService: EmailService;
+
+  constructor() {
+    this.rssFetcher = new NitterRssFetcher();
+    this.tweetProcessor = new TweetProcessor();
+    this.summaryGenerator = new SummaryGenerator();
+    this.emailService = new EmailService();
+  }
+
+  async run(): Promise<PipelineResult> {
+    const errors: PipelineError[] = [];
+    const startTime = Date.now();
+
+    logger.info('Starting newsletter pipeline');
+
+    try {
+      // Step 1: Fetch RSS feeds
+      logger.info('Step 1: Fetching RSS feeds');
+      const fetchResult = await this.rssFetcher.fetchAll();
+
+      for (const err of fetchResult.errors) {
+        errors.push({
+          stage: 'rss',
+          message: `Failed to fetch @${err.account}: ${err.error}`,
+        });
+      }
+
+      if (fetchResult.tweets.length === 0) {
+        throw new Error('No tweets fetched from any source');
+      }
+
+      logger.info({ tweetCount: fetchResult.tweets.length }, 'RSS fetch complete');
+
+      // Step 2: Process tweets
+      logger.info('Step 2: Processing tweets');
+      const processedTweets = this.tweetProcessor.process(fetchResult.tweets);
+      const tweetsByTopic = this.tweetProcessor.groupByTopic(processedTweets);
+
+      logger.info(
+        {
+          processedCount: processedTweets.length,
+          topics: Array.from(tweetsByTopic.keys()),
+        },
+        'Tweet processing complete'
+      );
+
+      // Step 3: Generate AI summaries
+      logger.info('Step 3: Generating AI summaries');
+      const topicSummaries = await this.generateSummaries(tweetsByTopic, errors);
+
+      // Step 4: Generate daily insights
+      logger.info('Step 4: Generating daily insights');
+      let insights: string;
+      try {
+        if (config.features.enableAiSummaries) {
+          insights = await this.summaryGenerator.generateDailyInsights(topicSummaries);
+        } else {
+          insights = this.createFallbackInsights(topicSummaries);
+        }
+      } catch (error) {
+        logger.error({ error }, 'Failed to generate insights');
+        errors.push({
+          stage: 'ai',
+          message: 'Failed to generate daily insights',
+          details: error,
+        });
+        insights = this.createFallbackInsights(topicSummaries);
+      }
+
+      // Step 5: Build newsletter
+      logger.info('Step 5: Building newsletter');
+      const newsletter: Newsletter = {
+        date: new Date(),
+        insights,
+        topics: topicSummaries,
+        totalTweets: processedTweets.length,
+        errors,
+      };
+
+      // Step 6: Send email
+      logger.info('Step 6: Sending email');
+      try {
+        const sendResult = await this.emailService.sendNewsletter(newsletter);
+
+        if (!sendResult.success) {
+          errors.push({
+            stage: 'email',
+            message: sendResult.error || 'Email send failed',
+          });
+        }
+      } catch (error) {
+        logger.error({ error }, 'Email sending failed');
+        errors.push({
+          stage: 'email',
+          message: 'Failed to send newsletter email',
+          details: error,
+        });
+      }
+
+      const duration = Date.now() - startTime;
+      logger.info(
+        {
+          durationMs: duration,
+          totalTweets: processedTweets.length,
+          topicCount: topicSummaries.length,
+          errorCount: errors.length,
+        },
+        'Newsletter pipeline completed'
+      );
+
+      return {
+        success: errors.filter((e) => e.stage === 'email').length === 0,
+        newsletter,
+        errors,
+      };
+    } catch (error) {
+      logger.error({ error }, 'Newsletter pipeline failed');
+
+      return {
+        success: false,
+        errors: [
+          ...errors,
+          {
+            stage: 'process',
+            message: error instanceof Error ? error.message : 'Unknown error',
+            details: error,
+          },
+        ],
+      };
+    }
+  }
+
+  private async generateSummaries(
+    tweetsByTopic: Map<TopicId, import('../types/index.js').ProcessedTweet[]>,
+    errors: PipelineError[]
+  ): Promise<TopicSummary[]> {
+    const summaries: TopicSummary[] = [];
+
+    for (const topic of TOPICS) {
+      const tweets = tweetsByTopic.get(topic.id) || [];
+
+      try {
+        if (config.features.enableAiSummaries) {
+          const summary = await this.summaryGenerator.generateTopicSummary(topic.id, tweets);
+          summaries.push(summary);
+        } else {
+          summaries.push(this.createBasicSummary(topic, tweets));
+        }
+      } catch (error) {
+        logger.error({ topic: topic.id, error }, 'Failed to generate topic summary');
+        errors.push({
+          stage: 'ai',
+          message: `Failed to generate summary for ${topic.name}`,
+          details: error,
+        });
+        summaries.push(this.createBasicSummary(topic, tweets));
+      }
+    }
+
+    return summaries;
+  }
+
+  private createBasicSummary(
+    topic: typeof TOPICS[number],
+    tweets: import('../types/index.js').ProcessedTweet[]
+  ): TopicSummary {
+    const uniqueAuthors = [...new Set(tweets.map((t) => t.authorDisplayName))];
+
+    return {
+      topic,
+      summary:
+        tweets.length > 0
+          ? `${tweets.length} posts from ${uniqueAuthors.slice(0, 3).join(', ')}${uniqueAuthors.length > 3 ? ' and others' : ''}.`
+          : `No ${topic.name} updates today.`,
+      highlights: tweets.slice(0, 3).map((t) => ({
+        tweet: t.content.slice(0, 200),
+        author: t.author,
+        context: `From ${t.authorDisplayName}`,
+        link: t.link,
+      })),
+      trends: [],
+      tweetCount: tweets.length,
+    };
+  }
+
+  private createFallbackInsights(summaries: TopicSummary[]): string {
+    const activeTopic = summaries.find((s) => s.tweetCount > 0);
+    if (!activeTopic) {
+      return 'A quiet day in tech - check back tomorrow!';
+    }
+
+    const topicNames = summaries
+      .filter((s) => s.tweetCount > 0)
+      .map((s) => s.topic.name)
+      .join(', ');
+
+    return `Today's tech discourse spans ${topicNames}. Dive into the highlights below!`;
+  }
+}

src/core/TweetProcessor.ts

@@ -0,0 +1,134 @@
+import { subHours, isAfter } from 'date-fns';
+import { config } from '../config/index.js';
+import { TECH_ACCOUNTS } from '../config/accounts.js';
+import { matchTopics } from '../config/topics.js';
+import { logger } from '../utils/logger.js';
+import type { RawTweet, ProcessedTweet, TopicId } from '../types/index.js';
+
+export class TweetProcessor {
+  private hoursLookback: number = 24;
+
+  process(rawTweets: RawTweet[]): ProcessedTweet[] {
+    logger.info({ input: rawTweets.length }, 'Processing tweets');
+
+    let tweets = rawTweets.map((tweet) => this.enrichTweet(tweet));
+
+    // Filter by time
+    const cutoff = subHours(new Date(), this.hoursLookback);
+    tweets = tweets.filter((t) => isAfter(t.timestamp, cutoff));
+    logger.debug({ afterTimeFilter: tweets.length }, 'Filtered by time');
+
+    // Filter retweets if configured
+    if (!config.features.includeRetweets) {
+      tweets = tweets.filter((t) => !t.isRetweet);
+      logger.debug({ afterRtFilter: tweets.length }, 'Filtered retweets');
+    }
+
+    // Filter replies if configured
+    if (!config.features.includeReplies) {
+      tweets = tweets.filter((t) => !t.isReply);
+      logger.debug({ afterReplyFilter: tweets.length }, 'Filtered replies');
+    }
+
+    // Deduplicate by content similarity
+    tweets = this.deduplicate(tweets);
+    logger.debug({ afterDedupe: tweets.length }, 'Deduplicated');
+
+    // Sort by relevance (high priority accounts first, then by time)
+    tweets = this.sortByRelevance(tweets);
+
+    logger.info({ output: tweets.length }, 'Tweet processing complete');
+
+    return tweets;
+  }
+
+  private enrichTweet(tweet: RawTweet): ProcessedTweet {
+    const content = tweet.content.toLowerCase();
+
+    // Detect retweets
+    const isRetweet = content.startsWith('rt @') || content.includes(' rt @');
+
+    // Detect replies (starts with @mention)
+    const isReply = tweet.content.trim().startsWith('@');
+
+    // Get account's default category
+    const account = TECH_ACCOUNTS.find((a) => a.username === tweet.author);
+    const accountCategory = account?.category;
+
+    // Match topics from content keywords
+    const contentTopics = matchTopics(tweet.content);
+
+    // Combine account category with content-matched topics
+    const topics: TopicId[] = accountCategory
+      ? [...new Set([accountCategory, ...contentTopics])]
+      : contentTopics.length > 0
+        ? contentTopics
+        : ['general_tech'];
+
+    return {
+      ...tweet,
+      topics,
+      isRetweet,
+      isReply,
+    };
+  }
+
+  private deduplicate(tweets: ProcessedTweet[]): ProcessedTweet[] {
+    const seen = new Map<string, ProcessedTweet>();
+
+    for (const tweet of tweets) {
+      const normalized = this.normalizeContent(tweet.content);
+
+      if (normalized.length < 20) continue;
+
+      const existing = seen.get(normalized);
+      if (!existing || tweet.timestamp > existing.timestamp) {
+        seen.set(normalized, tweet);
+      }
+    }
+
+    return Array.from(seen.values());
+  }
+
+  private normalizeContent(content: string): string {
+    return content
+      .toLowerCase()
+      .replace(/https?:\/\/\S+/g, '')
+      .replace(/@\w+/g, '')
+      .replace(/[^\w\s]/g, '')
+      .replace(/\s+/g, ' ')
+      .trim();
+  }
+
+  private sortByRelevance(tweets: ProcessedTweet[]): ProcessedTweet[] {
+    const priorityMap = new Map<string, number>();
+    for (const account of TECH_ACCOUNTS) {
+      priorityMap.set(account.username, account.priority === 'high' ? 3 : account.priority === 'medium' ? 2 : 1);
+    }
+
+    return tweets.sort((a, b) => {
+      const priorityA = priorityMap.get(a.author) || 0;
+      const priorityB = priorityMap.get(b.author) || 0;
+
+      if (priorityA !== priorityB) {
+        return priorityB - priorityA;
+      }
+
+      return b.timestamp.getTime() - a.timestamp.getTime();
+    });
+  }
+
+  groupByTopic(tweets: ProcessedTweet[]): Map<TopicId, ProcessedTweet[]> {
+    const grouped = new Map<TopicId, ProcessedTweet[]>();
+
+    for (const tweet of tweets) {
+      for (const topic of tweet.topics) {
+        const existing = grouped.get(topic) || [];
+        existing.push(tweet);
+        grouped.set(topic, existing);
+      }
+    }
+
+    return grouped;
+  }
+}

src/index.ts

@@ -0,0 +1,94 @@
+import { config } from './config/index.js';
+import { logger } from './utils/logger.js';
+import { NewsletterPipeline } from './core/NewsletterPipeline.js';
+import { CronScheduler } from './services/scheduler/CronScheduler.js';
+import { EmailService } from './services/email/EmailService.js';
+
+async function main() {
+  const args = process.argv.slice(2);
+  const runNow = args.includes('--run-now');
+  const dryRun = args.includes('--dry-run');
+
+  if (dryRun) {
+    process.env.DRY_RUN = 'true';
+  }
+
+  logger.info(
+    {
+      runNow,
+      dryRun: config.features.dryRun || dryRun,
+      cronSchedule: config.scheduler.cronExpression,
+      timezone: config.scheduler.timezone,
+      recipients: config.email.recipients.length,
+    },
+    'X-Newsletter starting'
+  );
+
+  const pipeline = new NewsletterPipeline();
+
+  if (runNow) {
+    logger.info('Running newsletter pipeline immediately');
+
+    const result = await pipeline.run();
+
+    if (result.success) {
+      logger.info('Newsletter sent successfully');
+      process.exit(0);
+    } else {
+      logger.error({ errors: result.errors }, 'Newsletter pipeline failed');
+      process.exit(1);
+    }
+  }
+
+  // Verify email connection on startup
+  const emailService = new EmailService();
+  const emailConnected = await emailService.verifyConnection();
+
+  if (!emailConnected) {
+    logger.warn('Email service connection could not be verified - will retry on send');
+  }
+
+  // Set up scheduled execution
+  const scheduler = new CronScheduler();
+
+  scheduler.schedule(
+    'daily-newsletter',
+    config.scheduler.cronExpression,
+    async () => {
+      const result = await pipeline.run();
+
+      if (!result.success) {
+        logger.error({ errors: result.errors }, 'Scheduled newsletter failed');
+      }
+    },
+    { timezone: config.scheduler.timezone }
+  );
+
+  scheduler.start();
+
+  logger.info(
+    {
+      schedule: config.scheduler.cronExpression,
+      timezone: config.scheduler.timezone,
+    },
+    'Newsletter scheduler started. Waiting for next scheduled run...'
+  );
+
+  // Handle graceful shutdown
+  const shutdown = () => {
+    logger.info('Shutting down...');
+    scheduler.stop();
+    process.exit(0);
+  };
+
+  process.on('SIGTERM', shutdown);
+  process.on('SIGINT', shutdown);
+
+  // Keep the process running
+  await new Promise(() => {});
+}
+
+main().catch((error) => {
+  logger.error({ error }, 'Fatal error');
+  process.exit(1);
+});

src/services/ai/OpenRouterClient.ts

@@ -0,0 +1,50 @@
+import OpenAI from 'openai';
+import { config } from '../../config/index.js';
+import { logger } from '../../utils/logger.js';
+import { withRetry } from '../../utils/retry.js';
+
+export class OpenRouterClient {
+  private client: OpenAI;
+  private model: string;
+  private maxTokens: number;
+
+  constructor() {
+    this.client = new OpenAI({
+      baseURL: 'https://openrouter.ai/api/v1',
+      apiKey: config.ai.openRouterApiKey,
+      defaultHeaders: {
+        'HTTP-Referer': config.ai.siteUrl,
+        'X-Title': config.ai.siteName,
+      },
+    });
+    this.model = config.ai.model;
+    this.maxTokens = config.ai.maxTokens;
+  }
+
+  async generateCompletion(prompt: string): Promise<string> {
+    logger.debug({ model: this.model, promptLength: prompt.length }, 'Generating AI completion');
+
+    return withRetry(
+      async () => {
+        const response = await this.client.chat.completions.create({
+          model: this.model,
+          messages: [{ role: 'user', content: prompt }],
+          max_tokens: this.maxTokens,
+          temperature: 0.7,
+        });
+
+        const content = response.choices[0]?.message?.content;
+        if (!content) {
+          throw new Error('Empty response from AI');
+        }
+
+        logger.debug({ responseLength: content.length }, 'AI completion generated');
+        return content;
+      },
+      {
+        maxAttempts: 3,
+        baseDelay: 2000,
+      }
+    );
+  }
+}

src/services/ai/SummaryGenerator.ts

@@ -0,0 +1,119 @@
+import { TOPICS } from '../../config/topics.js';
+import { logger } from '../../utils/logger.js';
+import type { ProcessedTweet, TopicSummary, TopicId, TweetHighlight } from '../../types/index.js';
+import { OpenRouterClient } from './OpenRouterClient.js';
+import { buildSummaryPrompt, buildInsightsPrompt, parseSummaryResponse } from './prompts.js';
+
+export class SummaryGenerator {
+  private client: OpenRouterClient;
+
+  constructor() {
+    this.client = new OpenRouterClient();
+  }
+
+  async generateTopicSummary(
+    topicId: TopicId,
+    tweets: ProcessedTweet[]
+  ): Promise<TopicSummary> {
+    const topic = TOPICS.find((t) => t.id === topicId);
+    if (!topic) {
+      throw new Error(`Unknown topic: ${topicId}`);
+    }
+
+    logger.info({ topic: topic.name, tweetCount: tweets.length }, 'Generating topic summary');
+
+    if (tweets.length === 0) {
+      return {
+        topic,
+        summary: `No significant ${topic.name} discussions today.`,
+        highlights: [],
+        trends: [],
+        tweetCount: 0,
+      };
+    }
+
+    try {
+      const prompt = buildSummaryPrompt(topic, tweets);
+      const response = await this.client.generateCompletion(prompt);
+      const parsed = parseSummaryResponse(response);
+
+      const highlights: TweetHighlight[] = parsed.highlights.map((h) => {
+        const matchingTweet = tweets.find(
+          (t) => t.author === h.author || t.content.includes(h.tweet.slice(0, 50))
+        );
+
+        return {
+          tweet: h.tweet,
+          author: h.author,
+          context: h.context,
+          link: matchingTweet?.link || `https://x.com/${h.author}`,
+        };
+      });
+
+      return {
+        topic,
+        summary: parsed.summary,
+        highlights,
+        trends: parsed.trends,
+        tweetCount: tweets.length,
+      };
+    } catch (error) {
+      logger.error({ topic: topic.name, error }, 'Failed to generate AI summary');
+      return this.createFallbackSummary(topic, tweets);
+    }
+  }
+
+  async generateDailyInsights(topicSummaries: TopicSummary[]): Promise<string> {
+    logger.info('Generating daily insights');
+
+    if (topicSummaries.length === 0 || topicSummaries.every((s) => s.tweetCount === 0)) {
+      return 'A quiet day in tech - stay tuned for tomorrow\'s updates!';
+    }
+
+    try {
+      const summaryTexts = topicSummaries
+        .filter((s) => s.tweetCount > 0)
+        .map((s) => `${s.topic.name}: ${s.summary}`);
+
+      const prompt = buildInsightsPrompt(summaryTexts);
+      const response = await this.client.generateCompletion(prompt);
+
+      return response.trim();
+    } catch (error) {
+      logger.error({ error }, 'Failed to generate daily insights');
+      return this.createFallbackInsights(topicSummaries);
+    }
+  }
+
+  private createFallbackSummary(
+    topic: typeof TOPICS[number],
+    tweets: ProcessedTweet[]
+  ): TopicSummary {
+    const topTweets = tweets.slice(0, 3);
+
+    return {
+      topic,
+      summary: `${tweets.length} discussions in ${topic.name} today from voices like ${[...new Set(tweets.slice(0, 5).map((t) => t.authorDisplayName))].join(', ')}.`,
+      highlights: topTweets.map((t) => ({
+        tweet: t.content.slice(0, 200),
+        author: t.author,
+        context: `Posted by ${t.authorDisplayName}`,
+        link: t.link,
+      })),
+      trends: [],
+      tweetCount: tweets.length,
+    };
+  }
+
+  private createFallbackInsights(topicSummaries: TopicSummary[]): string {
+    const topics = topicSummaries
+      .filter((s) => s.tweetCount > 0)
+      .map((s) => s.topic.name);
+
+    if (topics.length === 0) {
+      return 'Stay tuned for the next tech update!';
+    }
+
+    return `Today's tech discussions span ${topics.join(', ')}. Check out the highlights below!`;
+  }
+}

src/services/ai/prompts.ts

@@ -0,0 +1,80 @@
+import type { ProcessedTweet, TopicConfig } from '../../types/index.js';
+
+export function buildSummaryPrompt(topic: TopicConfig, tweets: ProcessedTweet[]): string {
+  const tweetList = tweets
+    .slice(0, 20)
+    .map(
+      (t, i) =>
+        `${i + 1}. @${t.author} (${t.authorDisplayName}): "${t.content.slice(0, 280)}"`
+    )
+    .join('\n');
+
+  return `You are a tech newsletter editor creating a daily digest about ${topic.name}.
+
+Analyze these tweets and provide:
+1. A concise summary (2-3 sentences) of the key themes and discussions
+2. The top 3 most important or interesting tweets with brief context explaining why they matter
+3. Any emerging trends or notable patterns
+
+Tweets:
+${tweetList}
+
+Respond ONLY with valid JSON in this exact format:
+{
+  "summary": "A 2-3 sentence summary of key themes...",
+  "highlights": [
+    {
+      "tweet": "The tweet content...",
+      "author": "username",
+      "context": "Why this tweet matters..."
+    }
+  ],
+  "trends": ["Trend 1", "Trend 2"]
+}`;
+}
+
+export function buildInsightsPrompt(topicSummaries: string[]): string {
+  return `You are a tech newsletter editor. Based on these topic summaries, provide a brief cross-topic insight (2-3 sentences) highlighting the most important themes of the day and any connections between different areas.
+
+Topic Summaries:
+${topicSummaries.join('\n\n')}
+
+Respond with just the insight text, no JSON or formatting. Keep it engaging and insightful.`;
+}
+
+export interface ParsedSummary {
+  summary: string;
+  highlights: {
+    tweet: string;
+    author: string;
+    context: string;
+  }[];
+  trends: string[];
+}
+
+export function parseSummaryResponse(response: string): ParsedSummary {
+  try {
+    const jsonMatch = response.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) {
+      throw new Error('No JSON found in response');
+    }
+
+    const parsed = JSON.parse(jsonMatch[0]);
+
+    if (!parsed.summary || !Array.isArray(parsed.highlights) || !Array.isArray(parsed.trends)) {
+      throw new Error('Invalid response structure');
+    }
+
+    return {
+      summary: parsed.summary,
+      highlights: parsed.highlights.slice(0, 3).map((h: Record<string, string>) => ({
+        tweet: h.tweet || '',
+        author: h.author || '',
+        context: h.context || '',
+      })),
+      trends: parsed.trends.slice(0, 5),
+    };
+  } catch (error) {
+    throw new Error(`Failed to parse AI response: ${error instanceof Error ? error.message : 'Unknown error'}`);
+  }
+}

src/services/email/EmailService.ts

@@ -0,0 +1,92 @@
+import nodemailer from 'nodemailer';
+import type { Transporter } from 'nodemailer';
+import { format } from 'date-fns';
+import { config } from '../../config/index.js';
+import { logger } from '../../utils/logger.js';
+import { withRetry } from '../../utils/retry.js';
+import type { Newsletter } from '../../types/index.js';
+import { renderNewsletterHtml } from './templates.js';
+
+export interface SendResult {
+  success: boolean;
+  messageId?: string;
+  accepted?: string[];
+  rejected?: string[];
+  error?: string;
+}
+
+export class EmailService {
+  private transporter: Transporter;
+
+  constructor() {
+    this.transporter = nodemailer.createTransport({
+      host: config.email.brevoHost,
+      port: config.email.brevoPort,
+      secure: false,
+      auth: {
+        user: config.email.brevoUser,
+        pass: config.email.brevoApiKey,
+      },
+    });
+  }
+
+  async sendNewsletter(newsletter: Newsletter): Promise<SendResult> {
+    const recipients = config.email.recipients;
+    const subject = `Daily Tech Digest - ${format(newsletter.date, 'MMM d, yyyy')}`;
+    const html = renderNewsletterHtml(newsletter);
+
+    logger.info({ recipients: recipients.length, subject }, 'Sending newsletter');
+
+    if (config.features.dryRun) {
+      logger.info('DRY RUN: Skipping email send');
+      return {
+        success: true,
+        messageId: 'dry-run',
+        accepted: recipients,
+        rejected: [],
+      };
+    }
+
+    return withRetry(
+      async () => {
+        const info = await this.transporter.sendMail({
+          from: `"${config.email.fromName}" <${config.email.fromEmail}>`,
+          to: recipients.join(', '),
+          subject,
+          html,
+        });
+
+        logger.info(
+          {
+            messageId: info.messageId,
+            accepted: info.accepted,
+            rejected: info.rejected,
+          },
+          'Newsletter sent successfully'
+        );
+
+        return {
+          success: true,
+          messageId: info.messageId,
+          accepted: info.accepted as string[],
+          rejected: info.rejected as string[],
+        };
+      },
+      {
+        maxAttempts: 3,
+        baseDelay: 5000,
+      }
+    );
+  }
+
+  async verifyConnection(): Promise<boolean> {
+    try {
+      await this.transporter.verify();
+      logger.info('Email service connection verified');
+      return true;
+    } catch (error) {
+      logger.error({ error }, 'Email service connection failed');
+      return false;
+    }
+  }
+}

src/services/email/templates.ts

@@ -0,0 +1,280 @@
+import { format } from 'date-fns';
+import type { Newsletter, TopicSummary, TweetHighlight } from '../../types/index.js';
+
+export function renderNewsletterHtml(newsletter: Newsletter): string {
+  const dateStr = format(newsletter.date, 'EEEE, MMMM d, yyyy');
+
+  return `
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <title>Daily Tech Digest - ${dateStr}</title>
+  <style>
+    body {
+      margin: 0;
+      padding: 0;
+      font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, sans-serif;
+      font-size: 16px;
+      line-height: 1.6;
+      color: #1a1a1a;
+      background-color: #f5f5f5;
+    }
+    .container {
+      max-width: 600px;
+      margin: 0 auto;
+      background-color: #ffffff;
+    }
+    .header {
+      background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+      color: #ffffff;
+      padding: 32px 24px;
+      text-align: center;
+    }
+    .header h1 {
+      margin: 0 0 8px 0;
+      font-size: 28px;
+      font-weight: 700;
+    }
+    .header .date {
+      font-size: 14px;
+      opacity: 0.9;
+    }
+    .content {
+      padding: 24px;
+    }
+    .insights-section {
+      background-color: #f8f9fa;
+      border-radius: 8px;
+      padding: 20px;
+      margin-bottom: 24px;
+      border-left: 4px solid #667eea;
+    }
+    .insights-section h2 {
+      margin: 0 0 12px 0;
+      font-size: 18px;
+      color: #667eea;
+    }
+    .insights-section p {
+      margin: 0;
+      color: #444;
+    }
+    .topic-section {
+      margin-bottom: 32px;
+    }
+    .topic-header {
+      display: flex;
+      align-items: center;
+      gap: 8px;
+      margin-bottom: 16px;
+      padding-bottom: 8px;
+      border-bottom: 2px solid #eee;
+    }
+    .topic-icon {
+      font-size: 24px;
+    }
+    .topic-title {
+      margin: 0;
+      font-size: 20px;
+      color: #333;
+    }
+    .topic-count {
+      font-size: 12px;
+      color: #888;
+      margin-left: auto;
+    }
+    .summary {
+      color: #444;
+      margin-bottom: 16px;
+    }
+    .highlights {
+      margin-top: 16px;
+    }
+    .highlight {
+      background-color: #fafafa;
+      border-radius: 8px;
+      padding: 16px;
+      margin-bottom: 12px;
+      border: 1px solid #eee;
+    }
+    .highlight-author {
+      font-weight: 600;
+      color: #667eea;
+      font-size: 14px;
+      margin-bottom: 8px;
+    }
+    .highlight-tweet {
+      color: #333;
+      margin-bottom: 8px;
+    }
+    .highlight-context {
+      font-size: 14px;
+      color: #666;
+      font-style: italic;
+    }
+    .highlight-link {
+      display: inline-block;
+      margin-top: 8px;
+      font-size: 13px;
+      color: #667eea;
+      text-decoration: none;
+    }
+    .highlight-link:hover {
+      text-decoration: underline;
+    }
+    .trends {
+      margin-top: 16px;
+    }
+    .trends-title {
+      font-size: 14px;
+      font-weight: 600;
+      color: #555;
+      margin-bottom: 8px;
+    }
+    .trend-tag {
+      display: inline-block;
+      background-color: #e8ecf1;
+      color: #555;
+      padding: 4px 12px;
+      border-radius: 16px;
+      font-size: 13px;
+      margin-right: 8px;
+      margin-bottom: 8px;
+    }
+    .footer {
+      background-color: #f8f9fa;
+      padding: 24px;
+      text-align: center;
+      font-size: 13px;
+      color: #666;
+    }
+    .footer a {
+      color: #667eea;
+      text-decoration: none;
+    }
+    .stats {
+      font-size: 12px;
+      color: #888;
+      margin-top: 8px;
+    }
+    @media only screen and (max-width: 600px) {
+      .content {
+        padding: 16px;
+      }
+      .header {
+        padding: 24px 16px;
+      }
+      .header h1 {
+        font-size: 24px;
+      }
+    }
+  </style>
+</head>
+<body>
+  <div class="container">
+    ${renderHeader(dateStr)}
+    <div class="content">
+      ${renderInsights(newsletter.insights)}
+      ${newsletter.topics.map(renderTopicSection).join('')}
+    </div>
+    ${renderFooter(newsletter)}
+  </div>
+</body>
+</html>
+  `.trim();
+}
+
+function renderHeader(dateStr: string): string {
+  return `
+    <div class="header">
+      <h1>Daily Tech Digest</h1>
+      <div class="date">${dateStr}</div>
+    </div>
+  `;
+}
+
+function renderInsights(insights: string): string {
+  return `
+    <div class="insights-section">
+      <h2>Today's Insights</h2>
+      <p>${escapeHtml(insights)}</p>
+    </div>
+  `;
+}
+
+function renderTopicSection(topic: TopicSummary): string {
+  if (topic.tweetCount === 0) {
+    return '';
+  }
+
+  return `
+    <div class="topic-section">
+      <div class="topic-header">
+        <span class="topic-icon">${topic.topic.icon}</span>
+        <h3 class="topic-title">${escapeHtml(topic.topic.name)}</h3>
+        <span class="topic-count">${topic.tweetCount} tweets</span>
+      </div>
+      <div class="summary">${escapeHtml(topic.summary)}</div>
+      ${topic.highlights.length > 0 ? renderHighlights(topic.highlights) : ''}
+      ${topic.trends.length > 0 ? renderTrends(topic.trends) : ''}
+    </div>
+  `;
+}
+
+function renderHighlights(highlights: TweetHighlight[]): string {
+  return `
+    <div class="highlights">
+      ${highlights.map(renderHighlight).join('')}
+    </div>
+  `;
+}
+
+function renderHighlight(highlight: TweetHighlight): string {
+  return `
+    <div class="highlight">
+      <div class="highlight-author">@${escapeHtml(highlight.author)}</div>
+      <div class="highlight-tweet">${escapeHtml(highlight.tweet)}</div>
+      <div class="highlight-context">${escapeHtml(highlight.context)}</div>
+      <a href="${escapeHtml(highlight.link)}" class="highlight-link" target="_blank">View on X &rarr;</a>
+    </div>
+  `;
+}
+
+function renderTrends(trends: string[]): string {
+  return `
+    <div class="trends">
+      <div class="trends-title">Trending</div>
+      ${trends.map((t) => `<span class="trend-tag">${escapeHtml(t)}</span>`).join('')}
+    </div>
+  `;
+}
+
+function renderFooter(newsletter: Newsletter): string {
+  const errorNote = newsletter.errors.length > 0
+    ? `<div class="stats">Note: ${newsletter.errors.length} data source(s) were unavailable</div>`
+    : '';
+
+  return `
+    <div class="footer">
+      <p>Curated with AI from ${newsletter.totalTweets} tweets</p>
+      ${errorNote}
+      <p style="margin-top: 16px;">
+        <a href="#">Unsubscribe</a> |
+        Generated by X-Newsletter
+      </p>
+    </div>
+  `;
+}
+
+function escapeHtml(text: string): string {
+  const map: Record<string, string> = {
+    '&': '&amp;',
+    '<': '&lt;',
+    '>': '&gt;',
+    '"': '&quot;',
+    "'": '&#039;',
+  };
+  return text.replace(/[&<>"']/g, (m) => map[m]);
+}

src/services/rss/NitterRssFetcher.ts

@@ -0,0 +1,164 @@
+import Parser from 'rss-parser';
+import axios from 'axios';
+import he from 'he';
+import sanitizeHtml from 'sanitize-html';
+import { config } from '../../config/index.js';
+import { TECH_ACCOUNTS } from '../../config/accounts.js';
+import { logger } from '../../utils/logger.js';
+import { withRetry } from '../../utils/retry.js';
+import type { RawTweet, TechAccount } from '../../types/index.js';
+import type { RssFetchResult, RssItem } from './types.js';
+
+export class NitterRssFetcher {
+  private instances: string[];
+  private currentInstanceIndex: number = 0;
+  private parser: Parser;
+  private timeout: number;
+
+  constructor() {
+    this.instances = config.rss.nitterInstances;
+    this.timeout = config.rss.fetchTimeout;
+    this.parser = new Parser({
+      timeout: this.timeout,
+      customFields: {
+        item: ['dc:creator', 'creator'],
+      },
+    });
+  }
+
+  async fetchAll(): Promise<RssFetchResult> {
+    const allTweets: RawTweet[] = [];
+    const errors: { account: string; error: string }[] = [];
+
+    logger.info({ accountCount: TECH_ACCOUNTS.length }, 'Starting RSS fetch for all accounts');
+
+    const results = await Promise.allSettled(
+      TECH_ACCOUNTS.map((account) => this.fetchAccount(account))
+    );
+
+    for (let i = 0; i < results.length; i++) {
+      const result = results[i];
+      const account = TECH_ACCOUNTS[i];
+
+      if (result.status === 'fulfilled') {
+        allTweets.push(...result.value);
+        logger.debug({ account: account.username, tweets: result.value.length }, 'Fetched tweets');
+      } else {
+        errors.push({
+          account: account.username,
+          error: result.reason?.message || 'Unknown error',
+        });
+        logger.warn({ account: account.username, error: result.reason?.message }, 'Failed to fetch');
+      }
+    }
+
+    logger.info(
+      { totalTweets: allTweets.length, errors: errors.length },
+      'Completed RSS fetch'
+    );
+
+    return {
+      tweets: allTweets,
+      errors,
+      source: this.getCurrentInstance(),
+    };
+  }
+
+  private async fetchAccount(account: TechAccount): Promise<RawTweet[]> {
+    return withRetry(
+      async () => {
+        const instance = this.getCurrentInstance();
+        const url = `${instance}/${account.username}/rss`;
+
+        logger.debug({ url, account: account.username }, 'Fetching RSS feed');
+
+        try {
+          const response = await axios.get(url, {
+            timeout: this.timeout,
+            headers: {
+              'User-Agent': 'Mozilla/5.0 (compatible; TechNewsletter/1.0)',
+              Accept: 'application/rss+xml, application/xml, text/xml',
+            },
+            validateStatus: (status) => status < 500,
+          });
+
+          if (response.status === 429) {
+            this.rotateInstance();
+            throw new Error('Rate limited, rotating instance');
+          }
+
+          if (response.status !== 200) {
+            throw new Error(`HTTP ${response.status}`);
+          }
+
+          const feed = await this.parser.parseString(response.data);
+          return this.parseFeedItems(feed.items, account);
+        } catch (error) {
+          if (axios.isAxiosError(error) && error.code === 'ECONNABORTED') {
+            this.rotateInstance();
+            throw new Error('Request timeout, rotating instance');
+          }
+          throw error;
+        }
+      },
+      {
+        maxAttempts: this.instances.length,
+        baseDelay: 500,
+        onRetry: () => this.rotateInstance(),
+      }
+    );
+  }
+
+  private parseFeedItems(items: RssItem[], account: TechAccount): RawTweet[] {
+    const maxTweets = config.rss.maxTweetsPerAccount;
+
+    return items.slice(0, maxTweets).map((item) => {
+      const content = this.cleanContent(item.content || item.contentSnippet || item.title || '');
+      const timestamp = item.isoDate ? new Date(item.isoDate) : new Date();
+
+      return {
+        id: item.guid || item.link || `${account.username}-${timestamp.getTime()}`,
+        content,
+        author: account.username,
+        authorDisplayName: account.displayName,
+        timestamp,
+        link: this.cleanLink(item.link || ''),
+      };
+    });
+  }
+
+  private cleanContent(html: string): string {
+    const stripped = sanitizeHtml(html, {
+      allowedTags: [],
+      allowedAttributes: {},
+    });
+
+    const decoded = he.decode(stripped);
+
+    return decoded
+      .replace(/\s+/g, ' ')
+      .replace(/^RT @\w+:\s*/i, '')
+      .trim();
+  }
+
+  private cleanLink(link: string): string {
+    try {
+      const url = new URL(link);
+      if (url.hostname.includes('nitter') || url.hostname.includes('xcancel')) {
+        return link.replace(url.origin, 'https://x.com');
+      }
+      return link;
+    } catch {
+      return link;
+    }
+  }
+
+  private getCurrentInstance(): string {
+    return this.instances[this.currentInstanceIndex];
+  }
+
+  private rotateInstance(): void {
+    this.currentInstanceIndex = (this.currentInstanceIndex + 1) % this.instances.length;
+    logger.debug({ newInstance: this.getCurrentInstance() }, 'Rotated to new Nitter instance');
+  }
+}

src/services/rss/types.ts

@@ -0,0 +1,25 @@
+import type { RawTweet } from '../../types/index.js';
+
+export interface RssFetchResult {
+  tweets: RawTweet[];
+  errors: { account: string; error: string }[];
+  source: string;
+}
+
+export interface RssItem {
+  title?: string;
+  content?: string;
+  contentSnippet?: string;
+  link?: string;
+  pubDate?: string;
+  isoDate?: string;
+  creator?: string;
+  guid?: string;
+}
+
+export interface RssFeed {
+  title?: string;
+  description?: string;
+  link?: string;
+  items: RssItem[];
+}

src/services/scheduler/CronScheduler.ts

@@ -0,0 +1,61 @@
+import cron, { ScheduledTask } from 'node-cron';
+import { logger } from '../../utils/logger.js';
+
+export class CronScheduler {
+  private jobs: Map<string, ScheduledTask> = new Map();
+
+  schedule(
+    name: string,
+    expression: string,
+    task: () => Promise<void>,
+    options?: { timezone?: string }
+  ): void {
+    if (!cron.validate(expression)) {
+      throw new Error(`Invalid cron expression: ${expression}`);
+    }
+
+    const job = cron.schedule(
+      expression,
+      async () => {
+        logger.info({ job: name }, 'Running scheduled task');
+        const startTime = Date.now();
+
+        try {
+          await task();
+          const duration = Date.now() - startTime;
+          logger.info({ job: name, durationMs: duration }, 'Scheduled task completed');
+        } catch (error) {
+          logger.error({ job: name, error }, 'Scheduled task failed');
+        }
+      },
+      {
+        timezone: options?.timezone,
+        scheduled: false,
+      }
+    );
+
+    this.jobs.set(name, job);
+    logger.info({ job: name, expression, timezone: options?.timezone }, 'Scheduled job registered');
+  }
+
+  start(): void {
+    for (const [name, job] of this.jobs) {
+      job.start();
+      logger.info({ job: name }, 'Job started');
+    }
+  }
+
+  stop(): void {
+    for (const [name, job] of this.jobs) {
+      job.stop();
+      logger.info({ job: name }, 'Job stopped');
+    }
+  }
+
+  getNextRun(name: string): Date | null {
+    const job = this.jobs.get(name);
+    if (!job) return null;
+
+    return null;
+  }
+}

src/types/index.ts

@@ -0,0 +1,108 @@
+export type TopicId = 'ai_ml' | 'swe' | 'general_tech';
+
+export interface TechAccount {
+  username: string;
+  displayName: string;
+  category: TopicId;
+  priority: 'high' | 'medium' | 'low';
+}
+
+export interface TopicConfig {
+  id: TopicId;
+  name: string;
+  keywords: string[];
+  icon: string;
+}
+
+export interface RawTweet {
+  id: string;
+  content: string;
+  author: string;
+  authorDisplayName: string;
+  timestamp: Date;
+  link: string;
+}
+
+export interface ProcessedTweet extends RawTweet {
+  topics: TopicId[];
+  isRetweet: boolean;
+  isReply: boolean;
+}
+
+export interface TopicSummary {
+  topic: TopicConfig;
+  summary: string;
+  highlights: TweetHighlight[];
+  trends: string[];
+  tweetCount: number;
+}
+
+export interface TweetHighlight {
+  tweet: string;
+  author: string;
+  context: string;
+  link: string;
+}
+
+export interface Newsletter {
+  date: Date;
+  insights: string;
+  topics: TopicSummary[];
+  totalTweets: number;
+  errors: PipelineError[];
+}
+
+export interface PipelineError {
+  stage: 'rss' | 'process' | 'ai' | 'email';
+  message: string;
+  details?: unknown;
+}
+
+export interface PipelineResult {
+  success: boolean;
+  newsletter?: Newsletter;
+  errors: PipelineError[];
+}
+
+export interface RssFetchResult {
+  tweets: RawTweet[];
+  errors: { account: string; error: string }[];
+  source: string;
+}
+
+export interface AppConfig {
+  rss: {
+    nitterInstances: string[];
+    fetchTimeout: number;
+    maxTweetsPerAccount: number;
+  };
+  ai: {
+    openRouterApiKey: string;
+    model: string;
+    maxTokens: number;
+    siteUrl: string;
+    siteName: string;
+  };
+  email: {
+    brevoHost: string;
+    brevoPort: number;
+    brevoUser: string;
+    brevoApiKey: string;
+    fromEmail: string;
+    fromName: string;
+    recipients: string[];
+  };
+  scheduler: {
+    cronExpression: string;
+    timezone: string;
+  };
+  features: {
+    enableAiSummaries: boolean;
+    includeRetweets: boolean;
+    includeReplies: boolean;
+    dryRun: boolean;
+  };
+  logging: {
+    level: string;
+  };
+}

src/utils/logger.ts

@@ -0,0 +1,22 @@
+import pino from 'pino';
+import { config } from '../config/index.js';
+
+export const logger = pino({
+  level: config.logging.level,
+  transport:
+    process.env.NODE_ENV === 'development'
+      ? {
+          target: 'pino-pretty',
+          options: {
+            colorize: true,
+            translateTime: 'SYS:standard',
+            ignore: 'pid,hostname',
+          },
+        }
+      : undefined,
+  base: {
+    service: 'x-newsletter',
+  },
+});
+
+export type Logger = typeof logger;

src/utils/retry.ts

@@ -0,0 +1,50 @@
+import { logger } from './logger.js';
+
+export interface RetryOptions {
+  maxAttempts?: number;
+  baseDelay?: number;
+  maxDelay?: number;
+  onRetry?: (attempt: number, error: Error) => void;
+}
+
+export async function withRetry<T>(
+  fn: () => Promise<T>,
+  options: RetryOptions = {}
+): Promise<T> {
+  const { maxAttempts = 3, baseDelay = 1000, maxDelay = 30000, onRetry } = options;
+
+  let lastError: Error | undefined;
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+
+      if (attempt === maxAttempts) {
+        logger.error({ attempt, error: lastError.message }, 'All retry attempts exhausted');
+        throw lastError;
+      }
+
+      const delay = Math.min(baseDelay * Math.pow(2, attempt - 1), maxDelay);
+      const jitter = Math.random() * 0.1 * delay;
+
+      logger.warn(
+        { attempt, maxAttempts, delay: delay + jitter, error: lastError.message },
+        'Retrying after failure'
+      );
+
+      if (onRetry) {
+        onRetry(attempt, lastError);
+      }
+
+      await sleep(delay + jitter);
+    }
+  }
+
+  throw lastError;
+}
+
+export function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}