Initial commit: Daily tech newsletter worker
A Node.js/TypeScript worker that sends daily AI-generated summaries of tech news from X/Twitter via Nitter RSS feeds. Features: - Fetches tweets from 40+ curated tech accounts via Nitter RSS - Filters and categorizes by topic (AI/ML, SWE, General Tech) - Generates AI summaries using OpenRouter (Claude/GPT-4) - Sends professional HTML email via Brevo SMTP - Runs on cron schedule inside Docker container - Instance rotation for Nitter reliability - Graceful degradation if AI fails Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
52
.env.example
Normal file
52
.env.example
Normal file
@@ -0,0 +1,52 @@
|
||||
# =============================================================================
|
||||
# APPLICATION
|
||||
# =============================================================================
|
||||
NODE_ENV=production
|
||||
LOG_LEVEL=info # debug, info, warn, error
|
||||
|
||||
# =============================================================================
|
||||
# RSS CONFIGURATION
|
||||
# =============================================================================
|
||||
# Comma-separated list of Nitter instances (will rotate on failure)
|
||||
NITTER_INSTANCES=https://nitter.poast.org,https://xcancel.com,https://nitter.privacydev.net
|
||||
RSS_FETCH_TIMEOUT=30000 # 30 seconds
|
||||
RSS_MAX_TWEETS_PER_ACCOUNT=50
|
||||
|
||||
# =============================================================================
|
||||
# OPENROUTER AI
|
||||
# =============================================================================
|
||||
OPENROUTER_API_KEY=sk-or-v1-your-key-here
|
||||
OPENROUTER_MODEL=anthropic/claude-3-sonnet-20240229
|
||||
# Alternative models: openai/gpt-4-turbo, anthropic/claude-3-opus, google/gemini-pro
|
||||
OPENROUTER_MAX_TOKENS=2000
|
||||
OPENROUTER_SITE_URL=https://your-newsletter.com
|
||||
OPENROUTER_SITE_NAME=Tech Newsletter
|
||||
|
||||
# =============================================================================
|
||||
# BREVO SMTP
|
||||
# =============================================================================
|
||||
BREVO_SMTP_HOST=smtp-relay.brevo.com
|
||||
BREVO_SMTP_PORT=587
|
||||
BREVO_SMTP_USER=your-brevo-login-email
|
||||
BREVO_SMTP_KEY=your-brevo-smtp-api-key
|
||||
EMAIL_FROM_ADDRESS=newsletter@yourdomain.com
|
||||
EMAIL_FROM_NAME=Daily Tech Digest
|
||||
# Comma-separated recipient list
|
||||
EMAIL_RECIPIENTS=you@example.com
|
||||
|
||||
# =============================================================================
|
||||
# SCHEDULER
|
||||
# =============================================================================
|
||||
# Cron expression: minute hour day month weekday
|
||||
# Default: Every day at 7:00 AM
|
||||
CRON_SCHEDULE=0 7 * * *
|
||||
CRON_TIMEZONE=Europe/Warsaw
|
||||
|
||||
# =============================================================================
|
||||
# FEATURE FLAGS
|
||||
# =============================================================================
|
||||
ENABLE_AI_SUMMARIES=true
|
||||
INCLUDE_RETWEETS=false
|
||||
INCLUDE_REPLIES=false
|
||||
# Set true to skip email sending (for testing)
|
||||
DRY_RUN=false
|
||||
35
.gitignore
vendored
Normal file
35
.gitignore
vendored
Normal file
@@ -0,0 +1,35 @@
|
||||
# Dependencies
|
||||
node_modules/
|
||||
|
||||
# Build output
|
||||
dist/
|
||||
|
||||
# Environment files
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
*.log
|
||||
npm-debug.log*
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Testing
|
||||
coverage/
|
||||
|
||||
# Misc
|
||||
*.tgz
|
||||
.cache/
|
||||
|
||||
# Claude Code
|
||||
.claude/
|
||||
42
docker/Dockerfile
Normal file
42
docker/Dockerfile
Normal file
@@ -0,0 +1,42 @@
|
||||
# =============================================================================
|
||||
# Build Stage
|
||||
# =============================================================================
|
||||
FROM node:20-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install dependencies first (better layer caching)
|
||||
COPY package*.json ./
|
||||
RUN npm ci
|
||||
|
||||
# Copy source and build
|
||||
COPY tsconfig.json ./
|
||||
COPY src ./src
|
||||
RUN npm run build
|
||||
|
||||
# Prune dev dependencies
|
||||
RUN npm prune --production
|
||||
|
||||
# =============================================================================
|
||||
# Production Stage
|
||||
# =============================================================================
|
||||
FROM node:20-alpine AS production
|
||||
|
||||
# Security: run as non-root user
|
||||
RUN addgroup -g 1001 -S nodejs && \
|
||||
adduser -S newsletter -u 1001
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy built application
|
||||
COPY --from=builder --chown=newsletter:nodejs /app/node_modules ./node_modules
|
||||
COPY --from=builder --chown=newsletter:nodejs /app/dist ./dist
|
||||
COPY --from=builder --chown=newsletter:nodejs /app/package.json ./
|
||||
|
||||
USER newsletter
|
||||
|
||||
# Set timezone (can be overridden via env)
|
||||
ENV TZ=Europe/Warsaw
|
||||
|
||||
# Default command
|
||||
CMD ["node", "dist/index.js"]
|
||||
31
docker/docker-compose.yml
Normal file
31
docker/docker-compose.yml
Normal file
@@ -0,0 +1,31 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
newsletter-worker:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile
|
||||
container_name: x-newsletter
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- ../.env
|
||||
environment:
|
||||
- NODE_ENV=production
|
||||
- TZ=${CRON_TIMEZONE:-Europe/Warsaw}
|
||||
volumes:
|
||||
# Optional: persist logs
|
||||
- ../logs:/app/logs
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
# Resource limits
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.5'
|
||||
memory: 512M
|
||||
reservations:
|
||||
cpus: '0.25'
|
||||
memory: 256M
|
||||
51
package.json
Normal file
51
package.json
Normal file
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"name": "x-newsletter",
|
||||
"version": "1.0.0",
|
||||
"description": "Daily tech newsletter from X/Twitter via Nitter RSS feeds with AI summaries",
|
||||
"main": "dist/index.js",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"start": "node dist/index.js",
|
||||
"dev": "tsx src/index.ts",
|
||||
"run-now": "tsx src/index.ts --run-now",
|
||||
"dry-run": "tsx src/index.ts --run-now --dry-run"
|
||||
},
|
||||
"keywords": [
|
||||
"newsletter",
|
||||
"twitter",
|
||||
"x",
|
||||
"rss",
|
||||
"ai",
|
||||
"automation"
|
||||
],
|
||||
"author": "",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"axios": "^1.7.9",
|
||||
"date-fns": "^4.1.0",
|
||||
"date-fns-tz": "^3.2.0",
|
||||
"dotenv": "^16.4.7",
|
||||
"he": "^1.2.0",
|
||||
"node-cron": "^3.0.3",
|
||||
"nodemailer": "^6.9.16",
|
||||
"openai": "^4.77.0",
|
||||
"pino": "^9.6.0",
|
||||
"rss-parser": "^3.13.0",
|
||||
"sanitize-html": "^2.14.0",
|
||||
"zod": "^3.24.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/he": "^1.2.3",
|
||||
"@types/node": "^22.10.5",
|
||||
"@types/node-cron": "^3.0.11",
|
||||
"@types/nodemailer": "^6.4.17",
|
||||
"@types/sanitize-html": "^2.13.0",
|
||||
"pino-pretty": "^13.0.0",
|
||||
"tsx": "^4.19.2",
|
||||
"typescript": "^5.7.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
}
|
||||
60
src/config/accounts.ts
Normal file
60
src/config/accounts.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
import type { TechAccount } from '../types/index.js';
|
||||
|
||||
export const TECH_ACCOUNTS: TechAccount[] = [
|
||||
// ===========================================
|
||||
// AI / Machine Learning
|
||||
// ===========================================
|
||||
{ username: 'karpathy', displayName: 'Andrej Karpathy', category: 'ai_ml', priority: 'high' },
|
||||
{ username: 'ylecun', displayName: 'Yann LeCun', category: 'ai_ml', priority: 'high' },
|
||||
{ username: 'AndrewYNg', displayName: 'Andrew Ng', category: 'ai_ml', priority: 'high' },
|
||||
{ username: 'sama', displayName: 'Sam Altman', category: 'ai_ml', priority: 'high' },
|
||||
{ username: 'demaboris', displayName: 'Demis Hassabis', category: 'ai_ml', priority: 'high' },
|
||||
{ username: 'goodaboris', displayName: 'Ian Goodfellow', category: 'ai_ml', priority: 'medium' },
|
||||
{ username: 'fchollet', displayName: 'François Chollet', category: 'ai_ml', priority: 'high' },
|
||||
{ username: 'EMostaque', displayName: 'Emad Mostaque', category: 'ai_ml', priority: 'medium' },
|
||||
{ username: 'JimFan', displayName: 'Jim Fan', category: 'ai_ml', priority: 'high' },
|
||||
{ username: 'gaborig', displayName: 'George Hotz', category: 'ai_ml', priority: 'medium' },
|
||||
{ username: 'ClaudeMcAI', displayName: 'Claude (Anthropic)', category: 'ai_ml', priority: 'medium' },
|
||||
{ username: 'OpenAI', displayName: 'OpenAI', category: 'ai_ml', priority: 'high' },
|
||||
{ username: 'AnthropicAI', displayName: 'Anthropic', category: 'ai_ml', priority: 'high' },
|
||||
{ username: 'huggingface', displayName: 'Hugging Face', category: 'ai_ml', priority: 'high' },
|
||||
|
||||
// ===========================================
|
||||
// Software Engineering / Dev Tools
|
||||
// ===========================================
|
||||
{ username: 'ThePrimeagen', displayName: 'ThePrimeagen', category: 'swe', priority: 'high' },
|
||||
{ username: 'kelseyhightower', displayName: 'Kelsey Hightower', category: 'swe', priority: 'high' },
|
||||
{ username: 'mitchellh', displayName: 'Mitchell Hashimoto', category: 'swe', priority: 'high' },
|
||||
{ username: 'tjholowaychuk', displayName: 'TJ Holowaychuk', category: 'swe', priority: 'medium' },
|
||||
{ username: 'addyosmani', displayName: 'Addy Osmani', category: 'swe', priority: 'medium' },
|
||||
{ username: 'sarah_edo', displayName: 'Sarah Drasner', category: 'swe', priority: 'medium' },
|
||||
{ username: 'dan_abramov', displayName: 'Dan Abramov', category: 'swe', priority: 'high' },
|
||||
{ username: 'swyx', displayName: 'swyx', category: 'swe', priority: 'medium' },
|
||||
{ username: 'kentcdodds', displayName: 'Kent C. Dodds', category: 'swe', priority: 'medium' },
|
||||
{ username: 'taborwang', displayName: 'Tanner Linsley', category: 'swe', priority: 'medium' },
|
||||
{ username: 'raaborig', displayName: 'Ryan Dahl', category: 'swe', priority: 'high' },
|
||||
{ username: 'vercel', displayName: 'Vercel', category: 'swe', priority: 'medium' },
|
||||
{ username: 'github', displayName: 'GitHub', category: 'swe', priority: 'medium' },
|
||||
|
||||
// ===========================================
|
||||
// General Tech / Startups
|
||||
// ===========================================
|
||||
{ username: 'levelsio', displayName: 'Pieter Levels', category: 'general_tech', priority: 'high' },
|
||||
{ username: 'paulg', displayName: 'Paul Graham', category: 'general_tech', priority: 'high' },
|
||||
{ username: 'naval', displayName: 'Naval Ravikant', category: 'general_tech', priority: 'high' },
|
||||
{ username: 'elaborig', displayName: 'Elon Musk', category: 'general_tech', priority: 'medium' },
|
||||
{ username: 'jason', displayName: 'Jason Calacanis', category: 'general_tech', priority: 'medium' },
|
||||
{ username: 'balajis', displayName: 'Balaji Srinivasan', category: 'general_tech', priority: 'medium' },
|
||||
{ username: 'pmarca', displayName: 'Marc Andreessen', category: 'general_tech', priority: 'medium' },
|
||||
{ username: 'aborig', displayName: 'DHH', category: 'general_tech', priority: 'high' },
|
||||
{ username: 'benedictevans', displayName: 'Benedict Evans', category: 'general_tech', priority: 'medium' },
|
||||
{ username: 'jason_f', displayName: 'Jason Fried', category: 'general_tech', priority: 'medium' },
|
||||
];
|
||||
|
||||
export function getAccountsByCategory(category: TechAccount['category']): TechAccount[] {
|
||||
return TECH_ACCOUNTS.filter((account) => account.category === category);
|
||||
}
|
||||
|
||||
export function getHighPriorityAccounts(): TechAccount[] {
|
||||
return TECH_ACCOUNTS.filter((account) => account.priority === 'high');
|
||||
}
|
||||
89
src/config/index.ts
Normal file
89
src/config/index.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import { config as dotenvConfig } from 'dotenv';
|
||||
import { z } from 'zod';
|
||||
import type { AppConfig } from '../types/index.js';
|
||||
|
||||
dotenvConfig();
|
||||
|
||||
const envSchema = z.object({
|
||||
NODE_ENV: z.enum(['development', 'production', 'test']).default('development'),
|
||||
LOG_LEVEL: z.enum(['debug', 'info', 'warn', 'error']).default('info'),
|
||||
|
||||
NITTER_INSTANCES: z.string().default('https://nitter.poast.org,https://xcancel.com'),
|
||||
RSS_FETCH_TIMEOUT: z.coerce.number().default(30000),
|
||||
RSS_MAX_TWEETS_PER_ACCOUNT: z.coerce.number().default(50),
|
||||
|
||||
OPENROUTER_API_KEY: z.string().min(1, 'OPENROUTER_API_KEY is required'),
|
||||
OPENROUTER_MODEL: z.string().default('anthropic/claude-3-sonnet-20240229'),
|
||||
OPENROUTER_MAX_TOKENS: z.coerce.number().default(2000),
|
||||
OPENROUTER_SITE_URL: z.string().default('https://tech-newsletter.local'),
|
||||
OPENROUTER_SITE_NAME: z.string().default('Tech Newsletter'),
|
||||
|
||||
BREVO_SMTP_HOST: z.string().default('smtp-relay.brevo.com'),
|
||||
BREVO_SMTP_PORT: z.coerce.number().default(587),
|
||||
BREVO_SMTP_USER: z.string().min(1, 'BREVO_SMTP_USER is required'),
|
||||
BREVO_SMTP_KEY: z.string().min(1, 'BREVO_SMTP_KEY is required'),
|
||||
EMAIL_FROM_ADDRESS: z.string().email('Invalid EMAIL_FROM_ADDRESS'),
|
||||
EMAIL_FROM_NAME: z.string().default('Daily Tech Digest'),
|
||||
EMAIL_RECIPIENTS: z.string().min(1, 'EMAIL_RECIPIENTS is required'),
|
||||
|
||||
CRON_SCHEDULE: z.string().default('0 7 * * *'),
|
||||
CRON_TIMEZONE: z.string().default('Europe/Warsaw'),
|
||||
|
||||
ENABLE_AI_SUMMARIES: z.coerce.boolean().default(true),
|
||||
INCLUDE_RETWEETS: z.coerce.boolean().default(false),
|
||||
INCLUDE_REPLIES: z.coerce.boolean().default(false),
|
||||
DRY_RUN: z.coerce.boolean().default(false),
|
||||
});
|
||||
|
||||
function loadConfig(): AppConfig {
|
||||
const result = envSchema.safeParse(process.env);
|
||||
|
||||
if (!result.success) {
|
||||
console.error('Configuration validation failed:');
|
||||
for (const error of result.error.errors) {
|
||||
console.error(` - ${error.path.join('.')}: ${error.message}`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const env = result.data;
|
||||
|
||||
return {
|
||||
rss: {
|
||||
nitterInstances: env.NITTER_INSTANCES.split(',').map((s) => s.trim()),
|
||||
fetchTimeout: env.RSS_FETCH_TIMEOUT,
|
||||
maxTweetsPerAccount: env.RSS_MAX_TWEETS_PER_ACCOUNT,
|
||||
},
|
||||
ai: {
|
||||
openRouterApiKey: env.OPENROUTER_API_KEY,
|
||||
model: env.OPENROUTER_MODEL,
|
||||
maxTokens: env.OPENROUTER_MAX_TOKENS,
|
||||
siteUrl: env.OPENROUTER_SITE_URL,
|
||||
siteName: env.OPENROUTER_SITE_NAME,
|
||||
},
|
||||
email: {
|
||||
brevoHost: env.BREVO_SMTP_HOST,
|
||||
brevoPort: env.BREVO_SMTP_PORT,
|
||||
brevoUser: env.BREVO_SMTP_USER,
|
||||
brevoApiKey: env.BREVO_SMTP_KEY,
|
||||
fromEmail: env.EMAIL_FROM_ADDRESS,
|
||||
fromName: env.EMAIL_FROM_NAME,
|
||||
recipients: env.EMAIL_RECIPIENTS.split(',').map((s) => s.trim()),
|
||||
},
|
||||
scheduler: {
|
||||
cronExpression: env.CRON_SCHEDULE,
|
||||
timezone: env.CRON_TIMEZONE,
|
||||
},
|
||||
features: {
|
||||
enableAiSummaries: env.ENABLE_AI_SUMMARIES,
|
||||
includeRetweets: env.INCLUDE_RETWEETS,
|
||||
includeReplies: env.INCLUDE_REPLIES,
|
||||
dryRun: env.DRY_RUN,
|
||||
},
|
||||
logging: {
|
||||
level: env.LOG_LEVEL,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const config = loadConfig();
|
||||
210
src/config/topics.ts
Normal file
210
src/config/topics.ts
Normal file
@@ -0,0 +1,210 @@
|
||||
import type { TopicConfig, TopicId } from '../types/index.js';
|
||||
|
||||
export const TOPICS: TopicConfig[] = [
|
||||
{
|
||||
id: 'ai_ml',
|
||||
name: 'AI & Machine Learning',
|
||||
keywords: [
|
||||
'gpt',
|
||||
'gpt-4',
|
||||
'gpt-5',
|
||||
'chatgpt',
|
||||
'llm',
|
||||
'large language model',
|
||||
'transformer',
|
||||
'neural network',
|
||||
'deep learning',
|
||||
'machine learning',
|
||||
'ml',
|
||||
'artificial intelligence',
|
||||
'ai',
|
||||
'openai',
|
||||
'anthropic',
|
||||
'claude',
|
||||
'gemini',
|
||||
'mistral',
|
||||
'llama',
|
||||
'fine-tuning',
|
||||
'fine tuning',
|
||||
'rag',
|
||||
'retrieval augmented',
|
||||
'embedding',
|
||||
'vector database',
|
||||
'prompt engineering',
|
||||
'prompt',
|
||||
'diffusion',
|
||||
'stable diffusion',
|
||||
'midjourney',
|
||||
'dall-e',
|
||||
'sora',
|
||||
'multimodal',
|
||||
'vision model',
|
||||
'nlp',
|
||||
'natural language',
|
||||
'rlhf',
|
||||
'reinforcement learning',
|
||||
'model training',
|
||||
'inference',
|
||||
'hugging face',
|
||||
'pytorch',
|
||||
'tensorflow',
|
||||
'jax',
|
||||
'agi',
|
||||
'superintelligence',
|
||||
'ai safety',
|
||||
'alignment',
|
||||
],
|
||||
icon: '🤖',
|
||||
},
|
||||
{
|
||||
id: 'swe',
|
||||
name: 'Software Engineering',
|
||||
keywords: [
|
||||
'programming',
|
||||
'coding',
|
||||
'software',
|
||||
'developer',
|
||||
'development',
|
||||
'typescript',
|
||||
'javascript',
|
||||
'python',
|
||||
'rust',
|
||||
'go',
|
||||
'golang',
|
||||
'java',
|
||||
'c++',
|
||||
'react',
|
||||
'vue',
|
||||
'angular',
|
||||
'svelte',
|
||||
'node',
|
||||
'nodejs',
|
||||
'deno',
|
||||
'bun',
|
||||
'next.js',
|
||||
'nextjs',
|
||||
'remix',
|
||||
'api',
|
||||
'rest',
|
||||
'graphql',
|
||||
'grpc',
|
||||
'microservices',
|
||||
'kubernetes',
|
||||
'k8s',
|
||||
'docker',
|
||||
'container',
|
||||
'devops',
|
||||
'ci/cd',
|
||||
'git',
|
||||
'github',
|
||||
'gitlab',
|
||||
'open source',
|
||||
'oss',
|
||||
'framework',
|
||||
'library',
|
||||
'package',
|
||||
'npm',
|
||||
'testing',
|
||||
'tdd',
|
||||
'clean code',
|
||||
'architecture',
|
||||
'design pattern',
|
||||
'refactoring',
|
||||
'database',
|
||||
'postgresql',
|
||||
'mysql',
|
||||
'mongodb',
|
||||
'redis',
|
||||
'aws',
|
||||
'azure',
|
||||
'gcp',
|
||||
'cloud',
|
||||
'serverless',
|
||||
'edge',
|
||||
'wasm',
|
||||
'webassembly',
|
||||
'vim',
|
||||
'neovim',
|
||||
'vscode',
|
||||
'ide',
|
||||
'terminal',
|
||||
'cli',
|
||||
],
|
||||
icon: '💻',
|
||||
},
|
||||
{
|
||||
id: 'general_tech',
|
||||
name: 'Tech & Startups',
|
||||
keywords: [
|
||||
'startup',
|
||||
'founder',
|
||||
'entrepreneur',
|
||||
'indie hacker',
|
||||
'indiehacker',
|
||||
'saas',
|
||||
'product',
|
||||
'launch',
|
||||
'shipped',
|
||||
'mvp',
|
||||
'funding',
|
||||
'vc',
|
||||
'venture capital',
|
||||
'seed',
|
||||
'series a',
|
||||
'ipo',
|
||||
'acquisition',
|
||||
'tech news',
|
||||
'silicon valley',
|
||||
'y combinator',
|
||||
'yc',
|
||||
'product hunt',
|
||||
'hacker news',
|
||||
'tech twitter',
|
||||
'remote work',
|
||||
'wfh',
|
||||
'crypto',
|
||||
'blockchain',
|
||||
'web3',
|
||||
'bitcoin',
|
||||
'ethereum',
|
||||
'nft',
|
||||
'defi',
|
||||
'fintech',
|
||||
'biotech',
|
||||
'climate tech',
|
||||
'hardware',
|
||||
'robotics',
|
||||
'automation',
|
||||
'productivity',
|
||||
'notion',
|
||||
'obsidian',
|
||||
'tech layoffs',
|
||||
'hiring',
|
||||
'tech jobs',
|
||||
'career',
|
||||
'salary',
|
||||
'compensation',
|
||||
],
|
||||
icon: '🚀',
|
||||
},
|
||||
];
|
||||
|
||||
export function getTopicById(id: TopicId): TopicConfig | undefined {
|
||||
return TOPICS.find((topic) => topic.id === id);
|
||||
}
|
||||
|
||||
export function matchTopics(text: string): TopicId[] {
|
||||
const lowerText = text.toLowerCase();
|
||||
const matched: TopicId[] = [];
|
||||
|
||||
for (const topic of TOPICS) {
|
||||
for (const keyword of topic.keywords) {
|
||||
if (lowerText.includes(keyword.toLowerCase())) {
|
||||
matched.push(topic.id);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return matched;
|
||||
}
|
||||
218
src/core/NewsletterPipeline.ts
Normal file
218
src/core/NewsletterPipeline.ts
Normal file
@@ -0,0 +1,218 @@
|
||||
import { config } from '../config/index.js';
|
||||
import { TOPICS } from '../config/topics.js';
|
||||
import { logger } from '../utils/logger.js';
|
||||
import { NitterRssFetcher } from '../services/rss/NitterRssFetcher.js';
|
||||
import { TweetProcessor } from './TweetProcessor.js';
|
||||
import { SummaryGenerator } from '../services/ai/SummaryGenerator.js';
|
||||
import { EmailService } from '../services/email/EmailService.js';
|
||||
import type {
|
||||
Newsletter,
|
||||
TopicSummary,
|
||||
PipelineResult,
|
||||
PipelineError,
|
||||
TopicId,
|
||||
} from '../types/index.js';
|
||||
|
||||
export class NewsletterPipeline {
|
||||
private rssFetcher: NitterRssFetcher;
|
||||
private tweetProcessor: TweetProcessor;
|
||||
private summaryGenerator: SummaryGenerator;
|
||||
private emailService: EmailService;
|
||||
|
||||
constructor() {
|
||||
this.rssFetcher = new NitterRssFetcher();
|
||||
this.tweetProcessor = new TweetProcessor();
|
||||
this.summaryGenerator = new SummaryGenerator();
|
||||
this.emailService = new EmailService();
|
||||
}
|
||||
|
||||
async run(): Promise<PipelineResult> {
|
||||
const errors: PipelineError[] = [];
|
||||
const startTime = Date.now();
|
||||
|
||||
logger.info('Starting newsletter pipeline');
|
||||
|
||||
try {
|
||||
// Step 1: Fetch RSS feeds
|
||||
logger.info('Step 1: Fetching RSS feeds');
|
||||
const fetchResult = await this.rssFetcher.fetchAll();
|
||||
|
||||
for (const err of fetchResult.errors) {
|
||||
errors.push({
|
||||
stage: 'rss',
|
||||
message: `Failed to fetch @${err.account}: ${err.error}`,
|
||||
});
|
||||
}
|
||||
|
||||
if (fetchResult.tweets.length === 0) {
|
||||
throw new Error('No tweets fetched from any source');
|
||||
}
|
||||
|
||||
logger.info({ tweetCount: fetchResult.tweets.length }, 'RSS fetch complete');
|
||||
|
||||
// Step 2: Process tweets
|
||||
logger.info('Step 2: Processing tweets');
|
||||
const processedTweets = this.tweetProcessor.process(fetchResult.tweets);
|
||||
const tweetsByTopic = this.tweetProcessor.groupByTopic(processedTweets);
|
||||
|
||||
logger.info(
|
||||
{
|
||||
processedCount: processedTweets.length,
|
||||
topics: Array.from(tweetsByTopic.keys()),
|
||||
},
|
||||
'Tweet processing complete'
|
||||
);
|
||||
|
||||
// Step 3: Generate AI summaries
|
||||
logger.info('Step 3: Generating AI summaries');
|
||||
const topicSummaries = await this.generateSummaries(tweetsByTopic, errors);
|
||||
|
||||
// Step 4: Generate daily insights
|
||||
logger.info('Step 4: Generating daily insights');
|
||||
let insights: string;
|
||||
try {
|
||||
if (config.features.enableAiSummaries) {
|
||||
insights = await this.summaryGenerator.generateDailyInsights(topicSummaries);
|
||||
} else {
|
||||
insights = this.createFallbackInsights(topicSummaries);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error({ error }, 'Failed to generate insights');
|
||||
errors.push({
|
||||
stage: 'ai',
|
||||
message: 'Failed to generate daily insights',
|
||||
details: error,
|
||||
});
|
||||
insights = this.createFallbackInsights(topicSummaries);
|
||||
}
|
||||
|
||||
// Step 5: Build newsletter
|
||||
logger.info('Step 5: Building newsletter');
|
||||
const newsletter: Newsletter = {
|
||||
date: new Date(),
|
||||
insights,
|
||||
topics: topicSummaries,
|
||||
totalTweets: processedTweets.length,
|
||||
errors,
|
||||
};
|
||||
|
||||
// Step 6: Send email
|
||||
logger.info('Step 6: Sending email');
|
||||
try {
|
||||
const sendResult = await this.emailService.sendNewsletter(newsletter);
|
||||
|
||||
if (!sendResult.success) {
|
||||
errors.push({
|
||||
stage: 'email',
|
||||
message: sendResult.error || 'Email send failed',
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error({ error }, 'Email sending failed');
|
||||
errors.push({
|
||||
stage: 'email',
|
||||
message: 'Failed to send newsletter email',
|
||||
details: error,
|
||||
});
|
||||
}
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
logger.info(
|
||||
{
|
||||
durationMs: duration,
|
||||
totalTweets: processedTweets.length,
|
||||
topicCount: topicSummaries.length,
|
||||
errorCount: errors.length,
|
||||
},
|
||||
'Newsletter pipeline completed'
|
||||
);
|
||||
|
||||
return {
|
||||
success: errors.filter((e) => e.stage === 'email').length === 0,
|
||||
newsletter,
|
||||
errors,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error({ error }, 'Newsletter pipeline failed');
|
||||
|
||||
return {
|
||||
success: false,
|
||||
errors: [
|
||||
...errors,
|
||||
{
|
||||
stage: 'process',
|
||||
message: error instanceof Error ? error.message : 'Unknown error',
|
||||
details: error,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private async generateSummaries(
|
||||
tweetsByTopic: Map<TopicId, import('../types/index.js').ProcessedTweet[]>,
|
||||
errors: PipelineError[]
|
||||
): Promise<TopicSummary[]> {
|
||||
const summaries: TopicSummary[] = [];
|
||||
|
||||
for (const topic of TOPICS) {
|
||||
const tweets = tweetsByTopic.get(topic.id) || [];
|
||||
|
||||
try {
|
||||
if (config.features.enableAiSummaries) {
|
||||
const summary = await this.summaryGenerator.generateTopicSummary(topic.id, tweets);
|
||||
summaries.push(summary);
|
||||
} else {
|
||||
summaries.push(this.createBasicSummary(topic, tweets));
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error({ topic: topic.id, error }, 'Failed to generate topic summary');
|
||||
errors.push({
|
||||
stage: 'ai',
|
||||
message: `Failed to generate summary for ${topic.name}`,
|
||||
details: error,
|
||||
});
|
||||
summaries.push(this.createBasicSummary(topic, tweets));
|
||||
}
|
||||
}
|
||||
|
||||
return summaries;
|
||||
}
|
||||
|
||||
private createBasicSummary(
|
||||
topic: typeof TOPICS[number],
|
||||
tweets: import('../types/index.js').ProcessedTweet[]
|
||||
): TopicSummary {
|
||||
const uniqueAuthors = [...new Set(tweets.map((t) => t.authorDisplayName))];
|
||||
|
||||
return {
|
||||
topic,
|
||||
summary:
|
||||
tweets.length > 0
|
||||
? `${tweets.length} posts from ${uniqueAuthors.slice(0, 3).join(', ')}${uniqueAuthors.length > 3 ? ' and others' : ''}.`
|
||||
: `No ${topic.name} updates today.`,
|
||||
highlights: tweets.slice(0, 3).map((t) => ({
|
||||
tweet: t.content.slice(0, 200),
|
||||
author: t.author,
|
||||
context: `From ${t.authorDisplayName}`,
|
||||
link: t.link,
|
||||
})),
|
||||
trends: [],
|
||||
tweetCount: tweets.length,
|
||||
};
|
||||
}
|
||||
|
||||
private createFallbackInsights(summaries: TopicSummary[]): string {
|
||||
const activeTopic = summaries.find((s) => s.tweetCount > 0);
|
||||
if (!activeTopic) {
|
||||
return 'A quiet day in tech - check back tomorrow!';
|
||||
}
|
||||
|
||||
const topicNames = summaries
|
||||
.filter((s) => s.tweetCount > 0)
|
||||
.map((s) => s.topic.name)
|
||||
.join(', ');
|
||||
|
||||
return `Today's tech discourse spans ${topicNames}. Dive into the highlights below!`;
|
||||
}
|
||||
}
|
||||
134
src/core/TweetProcessor.ts
Normal file
134
src/core/TweetProcessor.ts
Normal file
@@ -0,0 +1,134 @@
|
||||
import { subHours, isAfter } from 'date-fns';
|
||||
import { config } from '../config/index.js';
|
||||
import { TECH_ACCOUNTS } from '../config/accounts.js';
|
||||
import { matchTopics } from '../config/topics.js';
|
||||
import { logger } from '../utils/logger.js';
|
||||
import type { RawTweet, ProcessedTweet, TopicId } from '../types/index.js';
|
||||
|
||||
export class TweetProcessor {
|
||||
private hoursLookback: number = 24;
|
||||
|
||||
process(rawTweets: RawTweet[]): ProcessedTweet[] {
|
||||
logger.info({ input: rawTweets.length }, 'Processing tweets');
|
||||
|
||||
let tweets = rawTweets.map((tweet) => this.enrichTweet(tweet));
|
||||
|
||||
// Filter by time
|
||||
const cutoff = subHours(new Date(), this.hoursLookback);
|
||||
tweets = tweets.filter((t) => isAfter(t.timestamp, cutoff));
|
||||
logger.debug({ afterTimeFilter: tweets.length }, 'Filtered by time');
|
||||
|
||||
// Filter retweets if configured
|
||||
if (!config.features.includeRetweets) {
|
||||
tweets = tweets.filter((t) => !t.isRetweet);
|
||||
logger.debug({ afterRtFilter: tweets.length }, 'Filtered retweets');
|
||||
}
|
||||
|
||||
// Filter replies if configured
|
||||
if (!config.features.includeReplies) {
|
||||
tweets = tweets.filter((t) => !t.isReply);
|
||||
logger.debug({ afterReplyFilter: tweets.length }, 'Filtered replies');
|
||||
}
|
||||
|
||||
// Deduplicate by content similarity
|
||||
tweets = this.deduplicate(tweets);
|
||||
logger.debug({ afterDedupe: tweets.length }, 'Deduplicated');
|
||||
|
||||
// Sort by relevance (high priority accounts first, then by time)
|
||||
tweets = this.sortByRelevance(tweets);
|
||||
|
||||
logger.info({ output: tweets.length }, 'Tweet processing complete');
|
||||
|
||||
return tweets;
|
||||
}
|
||||
|
||||
private enrichTweet(tweet: RawTweet): ProcessedTweet {
|
||||
const content = tweet.content.toLowerCase();
|
||||
|
||||
// Detect retweets
|
||||
const isRetweet = content.startsWith('rt @') || content.includes(' rt @');
|
||||
|
||||
// Detect replies (starts with @mention)
|
||||
const isReply = tweet.content.trim().startsWith('@');
|
||||
|
||||
// Get account's default category
|
||||
const account = TECH_ACCOUNTS.find((a) => a.username === tweet.author);
|
||||
const accountCategory = account?.category;
|
||||
|
||||
// Match topics from content keywords
|
||||
const contentTopics = matchTopics(tweet.content);
|
||||
|
||||
// Combine account category with content-matched topics
|
||||
const topics: TopicId[] = accountCategory
|
||||
? [...new Set([accountCategory, ...contentTopics])]
|
||||
: contentTopics.length > 0
|
||||
? contentTopics
|
||||
: ['general_tech'];
|
||||
|
||||
return {
|
||||
...tweet,
|
||||
topics,
|
||||
isRetweet,
|
||||
isReply,
|
||||
};
|
||||
}
|
||||
|
||||
private deduplicate(tweets: ProcessedTweet[]): ProcessedTweet[] {
|
||||
const seen = new Map<string, ProcessedTweet>();
|
||||
|
||||
for (const tweet of tweets) {
|
||||
const normalized = this.normalizeContent(tweet.content);
|
||||
|
||||
if (normalized.length < 20) continue;
|
||||
|
||||
const existing = seen.get(normalized);
|
||||
if (!existing || tweet.timestamp > existing.timestamp) {
|
||||
seen.set(normalized, tweet);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(seen.values());
|
||||
}
|
||||
|
||||
private normalizeContent(content: string): string {
|
||||
return content
|
||||
.toLowerCase()
|
||||
.replace(/https?:\/\/\S+/g, '')
|
||||
.replace(/@\w+/g, '')
|
||||
.replace(/[^\w\s]/g, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
private sortByRelevance(tweets: ProcessedTweet[]): ProcessedTweet[] {
|
||||
const priorityMap = new Map<string, number>();
|
||||
for (const account of TECH_ACCOUNTS) {
|
||||
priorityMap.set(account.username, account.priority === 'high' ? 3 : account.priority === 'medium' ? 2 : 1);
|
||||
}
|
||||
|
||||
return tweets.sort((a, b) => {
|
||||
const priorityA = priorityMap.get(a.author) || 0;
|
||||
const priorityB = priorityMap.get(b.author) || 0;
|
||||
|
||||
if (priorityA !== priorityB) {
|
||||
return priorityB - priorityA;
|
||||
}
|
||||
|
||||
return b.timestamp.getTime() - a.timestamp.getTime();
|
||||
});
|
||||
}
|
||||
|
||||
groupByTopic(tweets: ProcessedTweet[]): Map<TopicId, ProcessedTweet[]> {
|
||||
const grouped = new Map<TopicId, ProcessedTweet[]>();
|
||||
|
||||
for (const tweet of tweets) {
|
||||
for (const topic of tweet.topics) {
|
||||
const existing = grouped.get(topic) || [];
|
||||
existing.push(tweet);
|
||||
grouped.set(topic, existing);
|
||||
}
|
||||
}
|
||||
|
||||
return grouped;
|
||||
}
|
||||
}
|
||||
94
src/index.ts
Normal file
94
src/index.ts
Normal file
@@ -0,0 +1,94 @@
|
||||
import { config } from './config/index.js';
|
||||
import { logger } from './utils/logger.js';
|
||||
import { NewsletterPipeline } from './core/NewsletterPipeline.js';
|
||||
import { CronScheduler } from './services/scheduler/CronScheduler.js';
|
||||
import { EmailService } from './services/email/EmailService.js';
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const runNow = args.includes('--run-now');
|
||||
const dryRun = args.includes('--dry-run');
|
||||
|
||||
if (dryRun) {
|
||||
process.env.DRY_RUN = 'true';
|
||||
}
|
||||
|
||||
logger.info(
|
||||
{
|
||||
runNow,
|
||||
dryRun: config.features.dryRun || dryRun,
|
||||
cronSchedule: config.scheduler.cronExpression,
|
||||
timezone: config.scheduler.timezone,
|
||||
recipients: config.email.recipients.length,
|
||||
},
|
||||
'X-Newsletter starting'
|
||||
);
|
||||
|
||||
const pipeline = new NewsletterPipeline();
|
||||
|
||||
if (runNow) {
|
||||
logger.info('Running newsletter pipeline immediately');
|
||||
|
||||
const result = await pipeline.run();
|
||||
|
||||
if (result.success) {
|
||||
logger.info('Newsletter sent successfully');
|
||||
process.exit(0);
|
||||
} else {
|
||||
logger.error({ errors: result.errors }, 'Newsletter pipeline failed');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify email connection on startup
|
||||
const emailService = new EmailService();
|
||||
const emailConnected = await emailService.verifyConnection();
|
||||
|
||||
if (!emailConnected) {
|
||||
logger.warn('Email service connection could not be verified - will retry on send');
|
||||
}
|
||||
|
||||
// Set up scheduled execution
|
||||
const scheduler = new CronScheduler();
|
||||
|
||||
scheduler.schedule(
|
||||
'daily-newsletter',
|
||||
config.scheduler.cronExpression,
|
||||
async () => {
|
||||
const result = await pipeline.run();
|
||||
|
||||
if (!result.success) {
|
||||
logger.error({ errors: result.errors }, 'Scheduled newsletter failed');
|
||||
}
|
||||
},
|
||||
{ timezone: config.scheduler.timezone }
|
||||
);
|
||||
|
||||
scheduler.start();
|
||||
|
||||
logger.info(
|
||||
{
|
||||
schedule: config.scheduler.cronExpression,
|
||||
timezone: config.scheduler.timezone,
|
||||
},
|
||||
'Newsletter scheduler started. Waiting for next scheduled run...'
|
||||
);
|
||||
|
||||
// Handle graceful shutdown
|
||||
const shutdown = () => {
|
||||
logger.info('Shutting down...');
|
||||
scheduler.stop();
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
process.on('SIGTERM', shutdown);
|
||||
process.on('SIGINT', shutdown);
|
||||
|
||||
// Keep the process running
|
||||
await new Promise(() => {});
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
logger.error({ error }, 'Fatal error');
|
||||
process.exit(1);
|
||||
});
|
||||
50
src/services/ai/OpenRouterClient.ts
Normal file
50
src/services/ai/OpenRouterClient.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
import OpenAI from 'openai';
|
||||
import { config } from '../../config/index.js';
|
||||
import { logger } from '../../utils/logger.js';
|
||||
import { withRetry } from '../../utils/retry.js';
|
||||
|
||||
export class OpenRouterClient {
|
||||
private client: OpenAI;
|
||||
private model: string;
|
||||
private maxTokens: number;
|
||||
|
||||
constructor() {
|
||||
this.client = new OpenAI({
|
||||
baseURL: 'https://openrouter.ai/api/v1',
|
||||
apiKey: config.ai.openRouterApiKey,
|
||||
defaultHeaders: {
|
||||
'HTTP-Referer': config.ai.siteUrl,
|
||||
'X-Title': config.ai.siteName,
|
||||
},
|
||||
});
|
||||
this.model = config.ai.model;
|
||||
this.maxTokens = config.ai.maxTokens;
|
||||
}
|
||||
|
||||
async generateCompletion(prompt: string): Promise<string> {
|
||||
logger.debug({ model: this.model, promptLength: prompt.length }, 'Generating AI completion');
|
||||
|
||||
return withRetry(
|
||||
async () => {
|
||||
const response = await this.client.chat.completions.create({
|
||||
model: this.model,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
max_tokens: this.maxTokens,
|
||||
temperature: 0.7,
|
||||
});
|
||||
|
||||
const content = response.choices[0]?.message?.content;
|
||||
if (!content) {
|
||||
throw new Error('Empty response from AI');
|
||||
}
|
||||
|
||||
logger.debug({ responseLength: content.length }, 'AI completion generated');
|
||||
return content;
|
||||
},
|
||||
{
|
||||
maxAttempts: 3,
|
||||
baseDelay: 2000,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
119
src/services/ai/SummaryGenerator.ts
Normal file
119
src/services/ai/SummaryGenerator.ts
Normal file
@@ -0,0 +1,119 @@
|
||||
import { TOPICS } from '../../config/topics.js';
|
||||
import { logger } from '../../utils/logger.js';
|
||||
import type { ProcessedTweet, TopicSummary, TopicId, TweetHighlight } from '../../types/index.js';
|
||||
import { OpenRouterClient } from './OpenRouterClient.js';
|
||||
import { buildSummaryPrompt, buildInsightsPrompt, parseSummaryResponse } from './prompts.js';
|
||||
|
||||
export class SummaryGenerator {
|
||||
private client: OpenRouterClient;
|
||||
|
||||
constructor() {
|
||||
this.client = new OpenRouterClient();
|
||||
}
|
||||
|
||||
async generateTopicSummary(
|
||||
topicId: TopicId,
|
||||
tweets: ProcessedTweet[]
|
||||
): Promise<TopicSummary> {
|
||||
const topic = TOPICS.find((t) => t.id === topicId);
|
||||
if (!topic) {
|
||||
throw new Error(`Unknown topic: ${topicId}`);
|
||||
}
|
||||
|
||||
logger.info({ topic: topic.name, tweetCount: tweets.length }, 'Generating topic summary');
|
||||
|
||||
if (tweets.length === 0) {
|
||||
return {
|
||||
topic,
|
||||
summary: `No significant ${topic.name} discussions today.`,
|
||||
highlights: [],
|
||||
trends: [],
|
||||
tweetCount: 0,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const prompt = buildSummaryPrompt(topic, tweets);
|
||||
const response = await this.client.generateCompletion(prompt);
|
||||
const parsed = parseSummaryResponse(response);
|
||||
|
||||
const highlights: TweetHighlight[] = parsed.highlights.map((h) => {
|
||||
const matchingTweet = tweets.find(
|
||||
(t) => t.author === h.author || t.content.includes(h.tweet.slice(0, 50))
|
||||
);
|
||||
|
||||
return {
|
||||
tweet: h.tweet,
|
||||
author: h.author,
|
||||
context: h.context,
|
||||
link: matchingTweet?.link || `https://x.com/${h.author}`,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
topic,
|
||||
summary: parsed.summary,
|
||||
highlights,
|
||||
trends: parsed.trends,
|
||||
tweetCount: tweets.length,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error({ topic: topic.name, error }, 'Failed to generate AI summary');
|
||||
return this.createFallbackSummary(topic, tweets);
|
||||
}
|
||||
}
|
||||
|
||||
async generateDailyInsights(topicSummaries: TopicSummary[]): Promise<string> {
|
||||
logger.info('Generating daily insights');
|
||||
|
||||
if (topicSummaries.length === 0 || topicSummaries.every((s) => s.tweetCount === 0)) {
|
||||
return 'A quiet day in tech - stay tuned for tomorrow\'s updates!';
|
||||
}
|
||||
|
||||
try {
|
||||
const summaryTexts = topicSummaries
|
||||
.filter((s) => s.tweetCount > 0)
|
||||
.map((s) => `${s.topic.name}: ${s.summary}`);
|
||||
|
||||
const prompt = buildInsightsPrompt(summaryTexts);
|
||||
const response = await this.client.generateCompletion(prompt);
|
||||
|
||||
return response.trim();
|
||||
} catch (error) {
|
||||
logger.error({ error }, 'Failed to generate daily insights');
|
||||
return this.createFallbackInsights(topicSummaries);
|
||||
}
|
||||
}
|
||||
|
||||
private createFallbackSummary(
|
||||
topic: typeof TOPICS[number],
|
||||
tweets: ProcessedTweet[]
|
||||
): TopicSummary {
|
||||
const topTweets = tweets.slice(0, 3);
|
||||
|
||||
return {
|
||||
topic,
|
||||
summary: `${tweets.length} discussions in ${topic.name} today from voices like ${[...new Set(tweets.slice(0, 5).map((t) => t.authorDisplayName))].join(', ')}.`,
|
||||
highlights: topTweets.map((t) => ({
|
||||
tweet: t.content.slice(0, 200),
|
||||
author: t.author,
|
||||
context: `Posted by ${t.authorDisplayName}`,
|
||||
link: t.link,
|
||||
})),
|
||||
trends: [],
|
||||
tweetCount: tweets.length,
|
||||
};
|
||||
}
|
||||
|
||||
private createFallbackInsights(topicSummaries: TopicSummary[]): string {
|
||||
const topics = topicSummaries
|
||||
.filter((s) => s.tweetCount > 0)
|
||||
.map((s) => s.topic.name);
|
||||
|
||||
if (topics.length === 0) {
|
||||
return 'Stay tuned for the next tech update!';
|
||||
}
|
||||
|
||||
return `Today's tech discussions span ${topics.join(', ')}. Check out the highlights below!`;
|
||||
}
|
||||
}
|
||||
80
src/services/ai/prompts.ts
Normal file
80
src/services/ai/prompts.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
import type { ProcessedTweet, TopicConfig } from '../../types/index.js';
|
||||
|
||||
export function buildSummaryPrompt(topic: TopicConfig, tweets: ProcessedTweet[]): string {
|
||||
const tweetList = tweets
|
||||
.slice(0, 20)
|
||||
.map(
|
||||
(t, i) =>
|
||||
`${i + 1}. @${t.author} (${t.authorDisplayName}): "${t.content.slice(0, 280)}"`
|
||||
)
|
||||
.join('\n');
|
||||
|
||||
return `You are a tech newsletter editor creating a daily digest about ${topic.name}.
|
||||
|
||||
Analyze these tweets and provide:
|
||||
1. A concise summary (2-3 sentences) of the key themes and discussions
|
||||
2. The top 3 most important or interesting tweets with brief context explaining why they matter
|
||||
3. Any emerging trends or notable patterns
|
||||
|
||||
Tweets:
|
||||
${tweetList}
|
||||
|
||||
Respond ONLY with valid JSON in this exact format:
|
||||
{
|
||||
"summary": "A 2-3 sentence summary of key themes...",
|
||||
"highlights": [
|
||||
{
|
||||
"tweet": "The tweet content...",
|
||||
"author": "username",
|
||||
"context": "Why this tweet matters..."
|
||||
}
|
||||
],
|
||||
"trends": ["Trend 1", "Trend 2"]
|
||||
}`;
|
||||
}
|
||||
|
||||
export function buildInsightsPrompt(topicSummaries: string[]): string {
|
||||
return `You are a tech newsletter editor. Based on these topic summaries, provide a brief cross-topic insight (2-3 sentences) highlighting the most important themes of the day and any connections between different areas.
|
||||
|
||||
Topic Summaries:
|
||||
${topicSummaries.join('\n\n')}
|
||||
|
||||
Respond with just the insight text, no JSON or formatting. Keep it engaging and insightful.`;
|
||||
}
|
||||
|
||||
export interface ParsedSummary {
|
||||
summary: string;
|
||||
highlights: {
|
||||
tweet: string;
|
||||
author: string;
|
||||
context: string;
|
||||
}[];
|
||||
trends: string[];
|
||||
}
|
||||
|
||||
export function parseSummaryResponse(response: string): ParsedSummary {
|
||||
try {
|
||||
const jsonMatch = response.match(/\{[\s\S]*\}/);
|
||||
if (!jsonMatch) {
|
||||
throw new Error('No JSON found in response');
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(jsonMatch[0]);
|
||||
|
||||
if (!parsed.summary || !Array.isArray(parsed.highlights) || !Array.isArray(parsed.trends)) {
|
||||
throw new Error('Invalid response structure');
|
||||
}
|
||||
|
||||
return {
|
||||
summary: parsed.summary,
|
||||
highlights: parsed.highlights.slice(0, 3).map((h: Record<string, string>) => ({
|
||||
tweet: h.tweet || '',
|
||||
author: h.author || '',
|
||||
context: h.context || '',
|
||||
})),
|
||||
trends: parsed.trends.slice(0, 5),
|
||||
};
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to parse AI response: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
}
|
||||
}
|
||||
92
src/services/email/EmailService.ts
Normal file
92
src/services/email/EmailService.ts
Normal file
@@ -0,0 +1,92 @@
|
||||
import nodemailer from 'nodemailer';
|
||||
import type { Transporter } from 'nodemailer';
|
||||
import { format } from 'date-fns';
|
||||
import { config } from '../../config/index.js';
|
||||
import { logger } from '../../utils/logger.js';
|
||||
import { withRetry } from '../../utils/retry.js';
|
||||
import type { Newsletter } from '../../types/index.js';
|
||||
import { renderNewsletterHtml } from './templates.js';
|
||||
|
||||
export interface SendResult {
|
||||
success: boolean;
|
||||
messageId?: string;
|
||||
accepted?: string[];
|
||||
rejected?: string[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export class EmailService {
|
||||
private transporter: Transporter;
|
||||
|
||||
constructor() {
|
||||
this.transporter = nodemailer.createTransport({
|
||||
host: config.email.brevoHost,
|
||||
port: config.email.brevoPort,
|
||||
secure: false,
|
||||
auth: {
|
||||
user: config.email.brevoUser,
|
||||
pass: config.email.brevoApiKey,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async sendNewsletter(newsletter: Newsletter): Promise<SendResult> {
|
||||
const recipients = config.email.recipients;
|
||||
const subject = `Daily Tech Digest - ${format(newsletter.date, 'MMM d, yyyy')}`;
|
||||
const html = renderNewsletterHtml(newsletter);
|
||||
|
||||
logger.info({ recipients: recipients.length, subject }, 'Sending newsletter');
|
||||
|
||||
if (config.features.dryRun) {
|
||||
logger.info('DRY RUN: Skipping email send');
|
||||
return {
|
||||
success: true,
|
||||
messageId: 'dry-run',
|
||||
accepted: recipients,
|
||||
rejected: [],
|
||||
};
|
||||
}
|
||||
|
||||
return withRetry(
|
||||
async () => {
|
||||
const info = await this.transporter.sendMail({
|
||||
from: `"${config.email.fromName}" <${config.email.fromEmail}>`,
|
||||
to: recipients.join(', '),
|
||||
subject,
|
||||
html,
|
||||
});
|
||||
|
||||
logger.info(
|
||||
{
|
||||
messageId: info.messageId,
|
||||
accepted: info.accepted,
|
||||
rejected: info.rejected,
|
||||
},
|
||||
'Newsletter sent successfully'
|
||||
);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
messageId: info.messageId,
|
||||
accepted: info.accepted as string[],
|
||||
rejected: info.rejected as string[],
|
||||
};
|
||||
},
|
||||
{
|
||||
maxAttempts: 3,
|
||||
baseDelay: 5000,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async verifyConnection(): Promise<boolean> {
|
||||
try {
|
||||
await this.transporter.verify();
|
||||
logger.info('Email service connection verified');
|
||||
return true;
|
||||
} catch (error) {
|
||||
logger.error({ error }, 'Email service connection failed');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
280
src/services/email/templates.ts
Normal file
280
src/services/email/templates.ts
Normal file
@@ -0,0 +1,280 @@
|
||||
import { format } from 'date-fns';
|
||||
import type { Newsletter, TopicSummary, TweetHighlight } from '../../types/index.js';
|
||||
|
||||
export function renderNewsletterHtml(newsletter: Newsletter): string {
|
||||
const dateStr = format(newsletter.date, 'EEEE, MMMM d, yyyy');
|
||||
|
||||
return `
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<title>Daily Tech Digest - ${dateStr}</title>
|
||||
<style>
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, sans-serif;
|
||||
font-size: 16px;
|
||||
line-height: 1.6;
|
||||
color: #1a1a1a;
|
||||
background-color: #f5f5f5;
|
||||
}
|
||||
.container {
|
||||
max-width: 600px;
|
||||
margin: 0 auto;
|
||||
background-color: #ffffff;
|
||||
}
|
||||
.header {
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: #ffffff;
|
||||
padding: 32px 24px;
|
||||
text-align: center;
|
||||
}
|
||||
.header h1 {
|
||||
margin: 0 0 8px 0;
|
||||
font-size: 28px;
|
||||
font-weight: 700;
|
||||
}
|
||||
.header .date {
|
||||
font-size: 14px;
|
||||
opacity: 0.9;
|
||||
}
|
||||
.content {
|
||||
padding: 24px;
|
||||
}
|
||||
.insights-section {
|
||||
background-color: #f8f9fa;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
margin-bottom: 24px;
|
||||
border-left: 4px solid #667eea;
|
||||
}
|
||||
.insights-section h2 {
|
||||
margin: 0 0 12px 0;
|
||||
font-size: 18px;
|
||||
color: #667eea;
|
||||
}
|
||||
.insights-section p {
|
||||
margin: 0;
|
||||
color: #444;
|
||||
}
|
||||
.topic-section {
|
||||
margin-bottom: 32px;
|
||||
}
|
||||
.topic-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
margin-bottom: 16px;
|
||||
padding-bottom: 8px;
|
||||
border-bottom: 2px solid #eee;
|
||||
}
|
||||
.topic-icon {
|
||||
font-size: 24px;
|
||||
}
|
||||
.topic-title {
|
||||
margin: 0;
|
||||
font-size: 20px;
|
||||
color: #333;
|
||||
}
|
||||
.topic-count {
|
||||
font-size: 12px;
|
||||
color: #888;
|
||||
margin-left: auto;
|
||||
}
|
||||
.summary {
|
||||
color: #444;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
.highlights {
|
||||
margin-top: 16px;
|
||||
}
|
||||
.highlight {
|
||||
background-color: #fafafa;
|
||||
border-radius: 8px;
|
||||
padding: 16px;
|
||||
margin-bottom: 12px;
|
||||
border: 1px solid #eee;
|
||||
}
|
||||
.highlight-author {
|
||||
font-weight: 600;
|
||||
color: #667eea;
|
||||
font-size: 14px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
.highlight-tweet {
|
||||
color: #333;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
.highlight-context {
|
||||
font-size: 14px;
|
||||
color: #666;
|
||||
font-style: italic;
|
||||
}
|
||||
.highlight-link {
|
||||
display: inline-block;
|
||||
margin-top: 8px;
|
||||
font-size: 13px;
|
||||
color: #667eea;
|
||||
text-decoration: none;
|
||||
}
|
||||
.highlight-link:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
.trends {
|
||||
margin-top: 16px;
|
||||
}
|
||||
.trends-title {
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
color: #555;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
.trend-tag {
|
||||
display: inline-block;
|
||||
background-color: #e8ecf1;
|
||||
color: #555;
|
||||
padding: 4px 12px;
|
||||
border-radius: 16px;
|
||||
font-size: 13px;
|
||||
margin-right: 8px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
.footer {
|
||||
background-color: #f8f9fa;
|
||||
padding: 24px;
|
||||
text-align: center;
|
||||
font-size: 13px;
|
||||
color: #666;
|
||||
}
|
||||
.footer a {
|
||||
color: #667eea;
|
||||
text-decoration: none;
|
||||
}
|
||||
.stats {
|
||||
font-size: 12px;
|
||||
color: #888;
|
||||
margin-top: 8px;
|
||||
}
|
||||
@media only screen and (max-width: 600px) {
|
||||
.content {
|
||||
padding: 16px;
|
||||
}
|
||||
.header {
|
||||
padding: 24px 16px;
|
||||
}
|
||||
.header h1 {
|
||||
font-size: 24px;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
${renderHeader(dateStr)}
|
||||
<div class="content">
|
||||
${renderInsights(newsletter.insights)}
|
||||
${newsletter.topics.map(renderTopicSection).join('')}
|
||||
</div>
|
||||
${renderFooter(newsletter)}
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
`.trim();
|
||||
}
|
||||
|
||||
function renderHeader(dateStr: string): string {
|
||||
return `
|
||||
<div class="header">
|
||||
<h1>Daily Tech Digest</h1>
|
||||
<div class="date">${dateStr}</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
function renderInsights(insights: string): string {
|
||||
return `
|
||||
<div class="insights-section">
|
||||
<h2>Today's Insights</h2>
|
||||
<p>${escapeHtml(insights)}</p>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
function renderTopicSection(topic: TopicSummary): string {
|
||||
if (topic.tweetCount === 0) {
|
||||
return '';
|
||||
}
|
||||
|
||||
return `
|
||||
<div class="topic-section">
|
||||
<div class="topic-header">
|
||||
<span class="topic-icon">${topic.topic.icon}</span>
|
||||
<h3 class="topic-title">${escapeHtml(topic.topic.name)}</h3>
|
||||
<span class="topic-count">${topic.tweetCount} tweets</span>
|
||||
</div>
|
||||
<div class="summary">${escapeHtml(topic.summary)}</div>
|
||||
${topic.highlights.length > 0 ? renderHighlights(topic.highlights) : ''}
|
||||
${topic.trends.length > 0 ? renderTrends(topic.trends) : ''}
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
function renderHighlights(highlights: TweetHighlight[]): string {
|
||||
return `
|
||||
<div class="highlights">
|
||||
${highlights.map(renderHighlight).join('')}
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
function renderHighlight(highlight: TweetHighlight): string {
|
||||
return `
|
||||
<div class="highlight">
|
||||
<div class="highlight-author">@${escapeHtml(highlight.author)}</div>
|
||||
<div class="highlight-tweet">${escapeHtml(highlight.tweet)}</div>
|
||||
<div class="highlight-context">${escapeHtml(highlight.context)}</div>
|
||||
<a href="${escapeHtml(highlight.link)}" class="highlight-link" target="_blank">View on X →</a>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
function renderTrends(trends: string[]): string {
|
||||
return `
|
||||
<div class="trends">
|
||||
<div class="trends-title">Trending</div>
|
||||
${trends.map((t) => `<span class="trend-tag">${escapeHtml(t)}</span>`).join('')}
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
function renderFooter(newsletter: Newsletter): string {
|
||||
const errorNote = newsletter.errors.length > 0
|
||||
? `<div class="stats">Note: ${newsletter.errors.length} data source(s) were unavailable</div>`
|
||||
: '';
|
||||
|
||||
return `
|
||||
<div class="footer">
|
||||
<p>Curated with AI from ${newsletter.totalTweets} tweets</p>
|
||||
${errorNote}
|
||||
<p style="margin-top: 16px;">
|
||||
<a href="#">Unsubscribe</a> |
|
||||
Generated by X-Newsletter
|
||||
</p>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
function escapeHtml(text: string): string {
|
||||
const map: Record<string, string> = {
|
||||
'&': '&',
|
||||
'<': '<',
|
||||
'>': '>',
|
||||
'"': '"',
|
||||
"'": ''',
|
||||
};
|
||||
return text.replace(/[&<>"']/g, (m) => map[m]);
|
||||
}
|
||||
164
src/services/rss/NitterRssFetcher.ts
Normal file
164
src/services/rss/NitterRssFetcher.ts
Normal file
@@ -0,0 +1,164 @@
|
||||
import Parser from 'rss-parser';
|
||||
import axios from 'axios';
|
||||
import he from 'he';
|
||||
import sanitizeHtml from 'sanitize-html';
|
||||
import { config } from '../../config/index.js';
|
||||
import { TECH_ACCOUNTS } from '../../config/accounts.js';
|
||||
import { logger } from '../../utils/logger.js';
|
||||
import { withRetry } from '../../utils/retry.js';
|
||||
import type { RawTweet, TechAccount } from '../../types/index.js';
|
||||
import type { RssFetchResult, RssItem } from './types.js';
|
||||
|
||||
export class NitterRssFetcher {
|
||||
private instances: string[];
|
||||
private currentInstanceIndex: number = 0;
|
||||
private parser: Parser;
|
||||
private timeout: number;
|
||||
|
||||
constructor() {
|
||||
this.instances = config.rss.nitterInstances;
|
||||
this.timeout = config.rss.fetchTimeout;
|
||||
this.parser = new Parser({
|
||||
timeout: this.timeout,
|
||||
customFields: {
|
||||
item: ['dc:creator', 'creator'],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async fetchAll(): Promise<RssFetchResult> {
|
||||
const allTweets: RawTweet[] = [];
|
||||
const errors: { account: string; error: string }[] = [];
|
||||
|
||||
logger.info({ accountCount: TECH_ACCOUNTS.length }, 'Starting RSS fetch for all accounts');
|
||||
|
||||
const results = await Promise.allSettled(
|
||||
TECH_ACCOUNTS.map((account) => this.fetchAccount(account))
|
||||
);
|
||||
|
||||
for (let i = 0; i < results.length; i++) {
|
||||
const result = results[i];
|
||||
const account = TECH_ACCOUNTS[i];
|
||||
|
||||
if (result.status === 'fulfilled') {
|
||||
allTweets.push(...result.value);
|
||||
logger.debug({ account: account.username, tweets: result.value.length }, 'Fetched tweets');
|
||||
} else {
|
||||
errors.push({
|
||||
account: account.username,
|
||||
error: result.reason?.message || 'Unknown error',
|
||||
});
|
||||
logger.warn({ account: account.username, error: result.reason?.message }, 'Failed to fetch');
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(
|
||||
{ totalTweets: allTweets.length, errors: errors.length },
|
||||
'Completed RSS fetch'
|
||||
);
|
||||
|
||||
return {
|
||||
tweets: allTweets,
|
||||
errors,
|
||||
source: this.getCurrentInstance(),
|
||||
};
|
||||
}
|
||||
|
||||
private async fetchAccount(account: TechAccount): Promise<RawTweet[]> {
|
||||
return withRetry(
|
||||
async () => {
|
||||
const instance = this.getCurrentInstance();
|
||||
const url = `${instance}/${account.username}/rss`;
|
||||
|
||||
logger.debug({ url, account: account.username }, 'Fetching RSS feed');
|
||||
|
||||
try {
|
||||
const response = await axios.get(url, {
|
||||
timeout: this.timeout,
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; TechNewsletter/1.0)',
|
||||
Accept: 'application/rss+xml, application/xml, text/xml',
|
||||
},
|
||||
validateStatus: (status) => status < 500,
|
||||
});
|
||||
|
||||
if (response.status === 429) {
|
||||
this.rotateInstance();
|
||||
throw new Error('Rate limited, rotating instance');
|
||||
}
|
||||
|
||||
if (response.status !== 200) {
|
||||
throw new Error(`HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
const feed = await this.parser.parseString(response.data);
|
||||
return this.parseFeedItems(feed.items, account);
|
||||
} catch (error) {
|
||||
if (axios.isAxiosError(error) && error.code === 'ECONNABORTED') {
|
||||
this.rotateInstance();
|
||||
throw new Error('Request timeout, rotating instance');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
{
|
||||
maxAttempts: this.instances.length,
|
||||
baseDelay: 500,
|
||||
onRetry: () => this.rotateInstance(),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
private parseFeedItems(items: RssItem[], account: TechAccount): RawTweet[] {
|
||||
const maxTweets = config.rss.maxTweetsPerAccount;
|
||||
|
||||
return items.slice(0, maxTweets).map((item) => {
|
||||
const content = this.cleanContent(item.content || item.contentSnippet || item.title || '');
|
||||
const timestamp = item.isoDate ? new Date(item.isoDate) : new Date();
|
||||
|
||||
return {
|
||||
id: item.guid || item.link || `${account.username}-${timestamp.getTime()}`,
|
||||
content,
|
||||
author: account.username,
|
||||
authorDisplayName: account.displayName,
|
||||
timestamp,
|
||||
link: this.cleanLink(item.link || ''),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
private cleanContent(html: string): string {
|
||||
const stripped = sanitizeHtml(html, {
|
||||
allowedTags: [],
|
||||
allowedAttributes: {},
|
||||
});
|
||||
|
||||
const decoded = he.decode(stripped);
|
||||
|
||||
return decoded
|
||||
.replace(/\s+/g, ' ')
|
||||
.replace(/^RT @\w+:\s*/i, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
private cleanLink(link: string): string {
|
||||
try {
|
||||
const url = new URL(link);
|
||||
if (url.hostname.includes('nitter') || url.hostname.includes('xcancel')) {
|
||||
return link.replace(url.origin, 'https://x.com');
|
||||
}
|
||||
return link;
|
||||
} catch {
|
||||
return link;
|
||||
}
|
||||
}
|
||||
|
||||
private getCurrentInstance(): string {
|
||||
return this.instances[this.currentInstanceIndex];
|
||||
}
|
||||
|
||||
private rotateInstance(): void {
|
||||
this.currentInstanceIndex = (this.currentInstanceIndex + 1) % this.instances.length;
|
||||
logger.debug({ newInstance: this.getCurrentInstance() }, 'Rotated to new Nitter instance');
|
||||
}
|
||||
}
|
||||
25
src/services/rss/types.ts
Normal file
25
src/services/rss/types.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
import type { RawTweet } from '../../types/index.js';
|
||||
|
||||
export interface RssFetchResult {
|
||||
tweets: RawTweet[];
|
||||
errors: { account: string; error: string }[];
|
||||
source: string;
|
||||
}
|
||||
|
||||
export interface RssItem {
|
||||
title?: string;
|
||||
content?: string;
|
||||
contentSnippet?: string;
|
||||
link?: string;
|
||||
pubDate?: string;
|
||||
isoDate?: string;
|
||||
creator?: string;
|
||||
guid?: string;
|
||||
}
|
||||
|
||||
export interface RssFeed {
|
||||
title?: string;
|
||||
description?: string;
|
||||
link?: string;
|
||||
items: RssItem[];
|
||||
}
|
||||
61
src/services/scheduler/CronScheduler.ts
Normal file
61
src/services/scheduler/CronScheduler.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
import cron, { ScheduledTask } from 'node-cron';
|
||||
import { logger } from '../../utils/logger.js';
|
||||
|
||||
export class CronScheduler {
|
||||
private jobs: Map<string, ScheduledTask> = new Map();
|
||||
|
||||
schedule(
|
||||
name: string,
|
||||
expression: string,
|
||||
task: () => Promise<void>,
|
||||
options?: { timezone?: string }
|
||||
): void {
|
||||
if (!cron.validate(expression)) {
|
||||
throw new Error(`Invalid cron expression: ${expression}`);
|
||||
}
|
||||
|
||||
const job = cron.schedule(
|
||||
expression,
|
||||
async () => {
|
||||
logger.info({ job: name }, 'Running scheduled task');
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
await task();
|
||||
const duration = Date.now() - startTime;
|
||||
logger.info({ job: name, durationMs: duration }, 'Scheduled task completed');
|
||||
} catch (error) {
|
||||
logger.error({ job: name, error }, 'Scheduled task failed');
|
||||
}
|
||||
},
|
||||
{
|
||||
timezone: options?.timezone,
|
||||
scheduled: false,
|
||||
}
|
||||
);
|
||||
|
||||
this.jobs.set(name, job);
|
||||
logger.info({ job: name, expression, timezone: options?.timezone }, 'Scheduled job registered');
|
||||
}
|
||||
|
||||
start(): void {
|
||||
for (const [name, job] of this.jobs) {
|
||||
job.start();
|
||||
logger.info({ job: name }, 'Job started');
|
||||
}
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
for (const [name, job] of this.jobs) {
|
||||
job.stop();
|
||||
logger.info({ job: name }, 'Job stopped');
|
||||
}
|
||||
}
|
||||
|
||||
getNextRun(name: string): Date | null {
|
||||
const job = this.jobs.get(name);
|
||||
if (!job) return null;
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
108
src/types/index.ts
Normal file
108
src/types/index.ts
Normal file
@@ -0,0 +1,108 @@
|
||||
export type TopicId = 'ai_ml' | 'swe' | 'general_tech';
|
||||
|
||||
export interface TechAccount {
|
||||
username: string;
|
||||
displayName: string;
|
||||
category: TopicId;
|
||||
priority: 'high' | 'medium' | 'low';
|
||||
}
|
||||
|
||||
export interface TopicConfig {
|
||||
id: TopicId;
|
||||
name: string;
|
||||
keywords: string[];
|
||||
icon: string;
|
||||
}
|
||||
|
||||
export interface RawTweet {
|
||||
id: string;
|
||||
content: string;
|
||||
author: string;
|
||||
authorDisplayName: string;
|
||||
timestamp: Date;
|
||||
link: string;
|
||||
}
|
||||
|
||||
export interface ProcessedTweet extends RawTweet {
|
||||
topics: TopicId[];
|
||||
isRetweet: boolean;
|
||||
isReply: boolean;
|
||||
}
|
||||
|
||||
export interface TopicSummary {
|
||||
topic: TopicConfig;
|
||||
summary: string;
|
||||
highlights: TweetHighlight[];
|
||||
trends: string[];
|
||||
tweetCount: number;
|
||||
}
|
||||
|
||||
export interface TweetHighlight {
|
||||
tweet: string;
|
||||
author: string;
|
||||
context: string;
|
||||
link: string;
|
||||
}
|
||||
|
||||
export interface Newsletter {
|
||||
date: Date;
|
||||
insights: string;
|
||||
topics: TopicSummary[];
|
||||
totalTweets: number;
|
||||
errors: PipelineError[];
|
||||
}
|
||||
|
||||
export interface PipelineError {
|
||||
stage: 'rss' | 'process' | 'ai' | 'email';
|
||||
message: string;
|
||||
details?: unknown;
|
||||
}
|
||||
|
||||
export interface PipelineResult {
|
||||
success: boolean;
|
||||
newsletter?: Newsletter;
|
||||
errors: PipelineError[];
|
||||
}
|
||||
|
||||
export interface RssFetchResult {
|
||||
tweets: RawTweet[];
|
||||
errors: { account: string; error: string }[];
|
||||
source: string;
|
||||
}
|
||||
|
||||
export interface AppConfig {
|
||||
rss: {
|
||||
nitterInstances: string[];
|
||||
fetchTimeout: number;
|
||||
maxTweetsPerAccount: number;
|
||||
};
|
||||
ai: {
|
||||
openRouterApiKey: string;
|
||||
model: string;
|
||||
maxTokens: number;
|
||||
siteUrl: string;
|
||||
siteName: string;
|
||||
};
|
||||
email: {
|
||||
brevoHost: string;
|
||||
brevoPort: number;
|
||||
brevoUser: string;
|
||||
brevoApiKey: string;
|
||||
fromEmail: string;
|
||||
fromName: string;
|
||||
recipients: string[];
|
||||
};
|
||||
scheduler: {
|
||||
cronExpression: string;
|
||||
timezone: string;
|
||||
};
|
||||
features: {
|
||||
enableAiSummaries: boolean;
|
||||
includeRetweets: boolean;
|
||||
includeReplies: boolean;
|
||||
dryRun: boolean;
|
||||
};
|
||||
logging: {
|
||||
level: string;
|
||||
};
|
||||
}
|
||||
22
src/utils/logger.ts
Normal file
22
src/utils/logger.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import pino from 'pino';
|
||||
import { config } from '../config/index.js';
|
||||
|
||||
export const logger = pino({
|
||||
level: config.logging.level,
|
||||
transport:
|
||||
process.env.NODE_ENV === 'development'
|
||||
? {
|
||||
target: 'pino-pretty',
|
||||
options: {
|
||||
colorize: true,
|
||||
translateTime: 'SYS:standard',
|
||||
ignore: 'pid,hostname',
|
||||
},
|
||||
}
|
||||
: undefined,
|
||||
base: {
|
||||
service: 'x-newsletter',
|
||||
},
|
||||
});
|
||||
|
||||
export type Logger = typeof logger;
|
||||
50
src/utils/retry.ts
Normal file
50
src/utils/retry.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
import { logger } from './logger.js';
|
||||
|
||||
export interface RetryOptions {
|
||||
maxAttempts?: number;
|
||||
baseDelay?: number;
|
||||
maxDelay?: number;
|
||||
onRetry?: (attempt: number, error: Error) => void;
|
||||
}
|
||||
|
||||
export async function withRetry<T>(
|
||||
fn: () => Promise<T>,
|
||||
options: RetryOptions = {}
|
||||
): Promise<T> {
|
||||
const { maxAttempts = 3, baseDelay = 1000, maxDelay = 30000, onRetry } = options;
|
||||
|
||||
let lastError: Error | undefined;
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error : new Error(String(error));
|
||||
|
||||
if (attempt === maxAttempts) {
|
||||
logger.error({ attempt, error: lastError.message }, 'All retry attempts exhausted');
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
const delay = Math.min(baseDelay * Math.pow(2, attempt - 1), maxDelay);
|
||||
const jitter = Math.random() * 0.1 * delay;
|
||||
|
||||
logger.warn(
|
||||
{ attempt, maxAttempts, delay: delay + jitter, error: lastError.message },
|
||||
'Retrying after failure'
|
||||
);
|
||||
|
||||
if (onRetry) {
|
||||
onRetry(attempt, lastError);
|
||||
}
|
||||
|
||||
await sleep(delay + jitter);
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
export function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
24
tsconfig.json
Normal file
24
tsconfig.json
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"lib": ["ES2022"],
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noImplicitReturns": true,
|
||||
"noFallthroughCasesInSwitch": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
Reference in New Issue
Block a user