Initial commit

packages/scraper/src/client/poe-ninja.ts

@@ -0,0 +1,232 @@
+import type {
+  PoeNinjaSearchResponse,
+  PoeNinjaOverviewResponse,
+  PoeNinjaDetailsResponse,
+} from '@poe2data/shared';
+import { CURRENT_LEAGUE } from '@poe2data/shared';
+
+const BASE_URL = 'https://poe.ninja/poe2/api/economy/exchange/current';
+
+// Cache for item details keyed by "category:id"
+const itemDetailsCache = new Map<string, { name: string; icon: string }>();
+
+// Simple rate limiter
+class RateLimiter {
+  private tokens: number;
+  private lastRefill: number;
+  private readonly maxTokens: number;
+  private readonly refillRate: number;
+
+  constructor(maxTokens = 5, refillRateMs = 1000) {
+    this.maxTokens = maxTokens;
+    this.refillRate = refillRateMs;
+    this.tokens = maxTokens;
+    this.lastRefill = Date.now();
+  }
+
+  private refill(): void {
+    const now = Date.now();
+    const elapsed = now - this.lastRefill;
+    const tokensToAdd = Math.floor(elapsed / this.refillRate);
+    if (tokensToAdd > 0) {
+      this.tokens = Math.min(this.maxTokens, this.tokens + tokensToAdd);
+      this.lastRefill = now;
+    }
+  }
+
+  async acquire(): Promise<void> {
+    this.refill();
+    if (this.tokens <= 0) {
+      const waitTime = this.refillRate - (Date.now() - this.lastRefill);
+      await new Promise((resolve) => setTimeout(resolve, waitTime));
+      this.refill();
+    }
+    this.tokens--;
+  }
+}
+
+export class PoeNinjaClient {
+  private rateLimiter: RateLimiter;
+  private league: string;
+
+  constructor(league: string = CURRENT_LEAGUE) {
+    this.rateLimiter = new RateLimiter(5, 500); // 5 requests per 500ms
+    this.league = league;
+  }
+
+  private async fetch<T>(url: string): Promise<T> {
+    await this.rateLimiter.acquire();
+
+    const response = await fetch(url, {
+      headers: {
+        'User-Agent': 'POE2Data Scraper/1.0',
+        Accept: 'application/json',
+      },
+    });
+
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    }
+
+    return response.json() as Promise<T>;
+  }
+
+  /**
+   * Get all items across all categories
+   */
+  async search(): Promise<PoeNinjaSearchResponse> {
+    const url = `${BASE_URL}/search?league=${encodeURIComponent(this.league)}`;
+    return this.fetch<PoeNinjaSearchResponse>(url);
+  }
+
+  /**
+   * Get items for a specific category
+   */
+  async getOverview(type: string): Promise<PoeNinjaOverviewResponse> {
+    const url = `${BASE_URL}/overview?league=${encodeURIComponent(this.league)}&type=${encodeURIComponent(type)}`;
+    return this.fetch<PoeNinjaOverviewResponse>(url);
+  }
+
+  /**
+   * Get detailed info for a specific item
+   */
+  async getDetails(type: string, id: string): Promise<PoeNinjaDetailsResponse> {
+    const url = `${BASE_URL}/details?league=${encodeURIComponent(this.league)}&type=${encodeURIComponent(type)}&id=${encodeURIComponent(id)}`;
+    return this.fetch<PoeNinjaDetailsResponse>(url);
+  }
+
+  /**
+   * Set the league for subsequent requests
+   */
+  setLeague(league: string): void {
+    this.league = league;
+  }
+
+  /**
+   * Get item name and icon from cache
+   * Returns ID as fallback name if not in cache
+   */
+  getItemInfo(
+    type: string,
+    id: string
+  ): { name: string; icon: string } {
+    const cacheKey = `${type}:${id}`;
+
+    // Check cache first
+    const cached = itemDetailsCache.get(cacheKey);
+    if (cached) {
+      return cached;
+    }
+
+    // Not in cache, use ID as fallback name
+    // Don't call API as the details endpoint uses different ID format
+    const fallback = { name: id, icon: '' };
+    itemDetailsCache.set(cacheKey, fallback);
+    return fallback;
+  }
+
+  /**
+   * Preload item names from search endpoint
+   * This populates the cache with all known items, using multiple key variations
+   */
+  async preloadItemNames(): Promise<void> {
+    console.log('Preloading item names from search endpoint...');
+    const searchData = await this.search();
+
+    let count = 0;
+    // The search response has items grouped by category type
+    for (const [category, searchItems] of Object.entries(searchData.items)) {
+      for (const item of searchItems) {
+        const itemInfo = {
+          name: item.name,
+          icon: item.icon || '',
+        };
+
+        // Generate multiple possible keys for this item
+        const keys = this.generatePossibleKeys(item.name);
+        for (const key of keys) {
+          const cacheKey = `${category}:${key}`;
+          if (!itemDetailsCache.has(cacheKey)) {
+            itemDetailsCache.set(cacheKey, itemInfo);
+            count++;
+          }
+        }
+      }
+    }
+    console.log(`Preloaded ${count} cache entries from ${Object.keys(searchData.items).length} categories`);
+  }
+
+  /**
+   * Generate possible abbreviated keys for an item name
+   * e.g., "Orb of Alchemy" -> ["orb-of-alchemy", "alchemy", "alch"]
+   */
+  private generatePossibleKeys(name: string): string[] {
+    const keys: string[] = [];
+    const lower = name.toLowerCase();
+
+    // Full slugified name
+    keys.push(this.slugify(name));
+
+    // Common POE abbreviation patterns
+    // "Orb of X" -> "x" (first word of X)
+    const orbOfMatch = lower.match(/^orb of (\w+)/);
+    if (orbOfMatch) {
+      keys.push(orbOfMatch[1]);
+      // Also add 4-5 letter prefixes as abbreviations
+      if (orbOfMatch[1].length > 4) {
+        keys.push(orbOfMatch[1].substring(0, 4));
+        keys.push(orbOfMatch[1].substring(0, 5));
+      }
+    }
+
+    // "X Orb" -> "x" (first word)
+    const xOrbMatch = lower.match(/^(\w+)(?:'s)? orb/);
+    if (xOrbMatch) {
+      keys.push(xOrbMatch[1]);
+    }
+
+    // "X's Y" -> "x", "y", "xs"
+    const possessiveMatch = lower.match(/^(\w+)'s (\w+)/);
+    if (possessiveMatch) {
+      keys.push(possessiveMatch[1]);
+      keys.push(possessiveMatch[2]);
+      keys.push(possessiveMatch[1] + 's');
+    }
+
+    // First word alone
+    const firstWord = lower.split(/\s+/)[0].replace(/[^a-z]/g, '');
+    if (firstWord.length >= 3) {
+      keys.push(firstWord);
+    }
+
+    // Special abbreviations for common items
+    const specialMappings: Record<string, string> = {
+      'gemcutter\'s prism': 'gcp',
+      'glassblower\'s bauble': 'bauble',
+      'armourer\'s scrap': 'scrap',
+      'blacksmith\'s whetstone': 'whetstone',
+      'scroll of wisdom': 'wisdom',
+      'orb of transmutation': 'transmute',
+      'orb of regret': 'regret',
+      'vaal orb': 'vaal',
+      'mirror of kalandra': 'mirror',
+      'regal orb': 'regal',
+      'arcanist\'s etcher': 'etcher',
+    };
+    if (specialMappings[lower]) {
+      keys.push(specialMappings[lower]);
+    }
+
+    return [...new Set(keys)];
+  }
+
+  /**
+   * Convert item name to URL slug format
+   */
+  private slugify(name: string): string {
+    return name
+      .toLowerCase()
+      .replace(/[^a-z0-9]+/g, '-')
+      .replace(/^-|-$/g, '');
+  }
+}

packages/scraper/src/index.ts

@@ -0,0 +1,86 @@
+import { PoeNinjaClient } from './client/poe-ninja.js';
+import { DataSaver } from './services/data-saver.js';
+import { Scheduler } from './scheduler.js';
+import { POE2_CATEGORIES, CURRENT_LEAGUE } from '@poe2data/shared';
+import { closeDatabase } from '@poe2data/database';
+import { itemNameMapper } from './services/item-name-mapper.js';
+
+const SCHEDULED_MODE = process.argv.includes('--scheduled') || process.argv.includes('-s');
+
+async function runOnce() {
+  console.log(`POE2 Data Scraper - League: ${CURRENT_LEAGUE}`);
+  console.log('='.repeat(50));
+
+  const client = new PoeNinjaClient();
+  const saver = new DataSaver();
+
+  try {
+    // Initialize item name mapper from search endpoint
+    console.log('Initializing item name mapper...');
+    const searchData = await client.search();
+    itemNameMapper.initialize(searchData);
+
+    const leagueId = await saver.ensureLeague(CURRENT_LEAGUE);
+    console.log(`League ID: ${leagueId}`);
+
+    for (const category of POE2_CATEGORIES) {
+      console.log(`\nScraping ${category.title} (${category.type})...`);
+
+      try {
+        const data = await client.getOverview(category.type);
+        await saver.saveCurrencyOverview(data, category.title, category.type, leagueId);
+
+        const sorted = [...data.lines]
+          .sort((a, b) => (b.primaryValue || 0) - (a.primaryValue || 0))
+          .slice(0, 5);
+
+        const itemMap = new Map(data.core.items.map((i) => [i.id, i]));
+
+        console.log(`  Top 5 by value:`);
+        for (const line of sorted) {
+          const item = itemMap.get(line.id);
+          const name = item?.name || line.id;
+          const change = line.sparkline?.totalChange ?? 0;
+          const changeStr = change >= 0 ? `+${change.toFixed(1)}%` : `${change.toFixed(1)}%`;
+          console.log(
+            `    ${name.substring(0, 30).padEnd(32)} ${line.primaryValue.toFixed(4)} div [7d: ${changeStr}]`
+          );
+        }
+      } catch (err) {
+        console.error(`  Failed to scrape ${category.title}:`, err);
+      }
+    }
+
+    console.log('\n' + '='.repeat(50));
+    console.log('Scraping completed successfully!');
+  } catch (error) {
+    console.error('Error:', error);
+    process.exit(1);
+  } finally {
+    await closeDatabase();
+  }
+}
+
+async function runScheduled() {
+  console.log(`POE2 Data Scraper - SCHEDULED MODE`);
+  console.log(`League: ${CURRENT_LEAGUE}`);
+  console.log('='.repeat(50));
+
+  const scheduler = new Scheduler();
+
+  // Handle shutdown
+  process.on('SIGINT', async () => {
+    console.log('\nShutting down scheduler...');
+    await closeDatabase();
+    process.exit(0);
+  });
+
+  scheduler.start();
+}
+
+// Main entry point
+if (SCHEDULED_MODE) {
+  runScheduled();
+} else {
+  runOnce();
+}

packages/scraper/src/scheduler.ts

@@ -0,0 +1,76 @@
+import cron from 'node-cron';
+import { PoeNinjaClient } from './client/poe-ninja.js';
+import { DataSaver } from './services/data-saver.js';
+import { POE2_CATEGORIES, CURRENT_LEAGUE } from '@poe2data/shared';
+import { itemNameMapper } from './services/item-name-mapper.js';
+
+export class Scheduler {
+  private client: PoeNinjaClient;
+  private saver: DataSaver;
+  private leagueId: number | null = null;
+  private isRunning = false;
+
+  constructor() {
+    this.client = new PoeNinjaClient();
+    this.saver = new DataSaver();
+  }
+
+  async initialize() {
+    // Initialize item name mapper from search endpoint
+    console.log('Initializing item name mapper...');
+    const searchData = await this.client.search();
+    itemNameMapper.initialize(searchData);
+
+    this.leagueId = await this.saver.ensureLeague(CURRENT_LEAGUE);
+    console.log(`Scheduler initialized for league: ${CURRENT_LEAGUE} (ID: ${this.leagueId})`);
+  }
+
+  async scrapeAll() {
+    if (this.isRunning) {
+      console.log('Scrape already in progress, skipping...');
+      return;
+    }
+
+    if (!this.leagueId) {
+      await this.initialize();
+    }
+
+    this.isRunning = true;
+    const startTime = Date.now();
+    console.log(`\n[${ new Date().toISOString() }] Starting scrape...`);
+
+    let successCount = 0;
+    let errorCount = 0;
+
+    for (const category of POE2_CATEGORIES) {
+      try {
+        const data = await this.client.getOverview(category.type);
+        await this.saver.saveCurrencyOverview(data, category.title, category.type, this.leagueId!);
+        successCount++;
+      } catch (err) {
+        console.error(`  Failed to scrape ${category.title}:`, err);
+        errorCount++;
+      }
+    }
+
+    const duration = ((Date.now() - startTime) / 1000).toFixed(1);
+    console.log(`Scrape completed in ${duration}s: ${successCount} success, ${errorCount} errors\n`);
+
+    this.isRunning = false;
+  }
+
+  start() {
+    console.log('Starting scheduler...');
+    console.log('Schedule: Every 30 minutes');
+
+    // Run immediately on start
+    this.scrapeAll();
+
+    // Schedule every 30 minutes
+    cron.schedule('*/30 * * * *', () => {
+      this.scrapeAll();
+    });
+
+    console.log('Scheduler started. Press Ctrl+C to stop.');
+  }
+}

packages/scraper/src/services/data-saver.ts

@@ -0,0 +1,141 @@
+import { eq, and } from 'drizzle-orm';
+import {
+  getDatabase,
+  leagues,
+  items,
+  snapshots,
+  priceHistory,
+} from '@poe2data/database';
+import type { PoeNinjaOverviewResponse } from '@poe2data/shared';
+import { CURRENT_LEAGUE } from '@poe2data/shared';
+import { itemNameMapper, type ItemNameMapper } from './item-name-mapper.js';
+
+export class DataSaver {
+  private db = getDatabase();
+  private mapper: ItemNameMapper = itemNameMapper;
+
+  /**
+   * Ensure the league exists in the database
+   */
+  async ensureLeague(leagueName: string = CURRENT_LEAGUE): Promise<number> {
+    const existing = await this.db
+      .select()
+      .from(leagues)
+      .where(eq(leagues.name, leagueName))
+      .get();
+
+    if (existing) {
+      return existing.id;
+    }
+
+    const result = await this.db
+      .insert(leagues)
+      .values({
+        name: leagueName,
+        displayName: leagueName,
+        isActive: true,
+      })
+      .returning({ id: leagues.id });
+
+    return result[0].id;
+  }
+
+  /**
+   * Save currency overview data to the database
+   */
+  async saveCurrencyOverview(
+    data: PoeNinjaOverviewResponse,
+    category: string,
+    categoryType: string,
+    leagueId: number
+  ): Promise<void> {
+    // Create snapshot
+    const [snapshot] = await this.db
+      .insert(snapshots)
+      .values({
+        leagueId,
+        category,
+        status: 'success',
+        itemCount: data.lines.length,
+      })
+      .returning();
+
+    // Create item map from core.items (only has reference currencies)
+    const coreItemMap = new Map(
+      data.core.items.map((item) => [item.id, item])
+    );
+
+    let savedCount = 0;
+
+    // Process each line
+    for (const line of data.lines) {
+      // First try to get info from core.items
+      let itemName: string;
+      let itemIcon: string;
+      let detailsId: string;
+
+      const coreItem = coreItemMap.get(line.id);
+      if (coreItem) {
+        itemName = coreItem.name;
+        // Ensure full icon URL
+        itemIcon = coreItem.image.startsWith('http')
+          ? coreItem.image
+          : `https://web.poecdn.com${coreItem.image}`;
+        detailsId = coreItem.detailsId;
+      } else {
+        // Get from name mapper
+        const info = this.mapper.getItemInfo(categoryType, line.id);
+        itemName = info.name;
+        itemIcon = info.icon; // Already full URL from search endpoint
+        detailsId = line.id;
+      }
+
+      // Upsert item
+      let item = await this.db
+        .select()
+        .from(items)
+        .where(
+          and(
+            eq(items.externalId, line.id),
+            eq(items.leagueId, leagueId)
+          )
+        )
+        .get();
+
+      if (!item) {
+        const [newItem] = await this.db
+          .insert(items)
+          .values({
+            externalId: line.id,
+            detailsId,
+            leagueId,
+            category,
+            name: itemName,
+            iconUrl: itemIcon,
+          })
+          .returning();
+        item = newItem;
+      }
+
+      // Insert price history
+      await this.db.insert(priceHistory).values({
+        itemId: item.id,
+        snapshotId: snapshot.id,
+        divineValue: line.primaryValue,
+        volume: line.volumePrimaryValue,
+        change7d: line.sparkline?.totalChange,
+        sparklineData: line.sparkline?.data
+          ? JSON.stringify(line.sparkline.data)
+          : null,
+        exaltedRate: data.core.rates.exalted,
+        chaosRate: data.core.rates.chaos,
+      });
+
+      savedCount++;
+    }
+
+    console.log(
+      `Saved ${savedCount}/${data.lines.length} items for ${category} (snapshot #${snapshot.id})`
+    );
+  }
+}

packages/scraper/src/services/item-name-mapper.ts

@@ -0,0 +1,206 @@
+import type { PoeNinjaSearchResponse } from '@poe2data/shared';
+
+export interface ItemNameInfo {
+  name: string;
+  icon: string;
+}
+
+/**
+ * Service to map poe.ninja line IDs to proper item names
+ * The search endpoint returns items with names, the overview returns IDs
+ * This service builds a mapping between them
+ */
+export class ItemNameMapper {
+  private nameMap = new Map<string, ItemNameInfo>();
+  private initialized = false;
+
+  /**
+   * Initialize the mapper from search response data
+   */
+  initialize(searchData: PoeNinjaSearchResponse): void {
+    this.nameMap.clear();
+
+    for (const [category, items] of Object.entries(searchData.items)) {
+      for (const item of items) {
+        // Generate all possible ID variations for this item
+        const possibleIds = this.generatePossibleIds(item.name);
+
+        for (const id of possibleIds) {
+          const key = `${category}:${id}`;
+          if (!this.nameMap.has(key)) {
+            this.nameMap.set(key, {
+              name: item.name,
+              icon: item.icon || '',
+            });
+          }
+        }
+      }
+    }
+
+    this.initialized = true;
+    console.log(`ItemNameMapper initialized with ${this.nameMap.size} entries`);
+  }
+
+  /**
+   * Get item info by category and ID
+   */
+  getItemInfo(category: string, id: string): ItemNameInfo {
+    const key = `${category}:${id}`;
+    const info = this.nameMap.get(key);
+
+    if (info) {
+      return info;
+    }
+
+    // Try without category (some items might be miscategorized)
+    for (const [mapKey, mapInfo] of this.nameMap) {
+      if (mapKey.endsWith(`:${id}`)) {
+        return mapInfo;
+      }
+    }
+
+    // Return ID as fallback
+    return { name: this.formatIdAsName(id), icon: '' };
+  }
+
+  /**
+   * Check if mapper is initialized
+   */
+  isInitialized(): boolean {
+    return this.initialized;
+  }
+
+  /**
+   * Generate all possible ID variations from an item name
+   * poe.ninja uses various abbreviation schemes
+   */
+  private generatePossibleIds(name: string): string[] {
+    const ids: string[] = [];
+    const lower = name.toLowerCase();
+
+    // Full slugified name (e.g., "divine-orb")
+    ids.push(this.slugify(name));
+
+    // === CURRENCY PATTERNS ===
+
+    // "Orb of X" -> shortened forms
+    const orbOfMatch = lower.match(/^orb of (\w+)/);
+    if (orbOfMatch) {
+      const word = orbOfMatch[1];
+      ids.push(word);
+      // Common abbreviations: first 4-5 letters
+      if (word.length > 4) {
+        ids.push(word.substring(0, 4));
+        ids.push(word.substring(0, 5));
+      }
+      // "alchemy" -> "alch"
+      if (word === 'alchemy') ids.push('alch');
+      if (word === 'annulment') ids.push('annul');
+      if (word === 'augmentation') ids.push('aug');
+      if (word === 'transmutation') ids.push('transmute');
+      if (word === 'alteration') ids.push('alt');
+      if (word === 'regret') ids.push('regret');
+    }
+
+    // "X Orb" -> first word (e.g., "Chaos Orb" -> "chaos")
+    const xOrbMatch = lower.match(/^(\w+)(?:'s)? orb$/);
+    if (xOrbMatch) {
+      ids.push(xOrbMatch[1]);
+    }
+
+    // "X's Y" patterns (e.g., "Artificer's Orb" -> "artificers")
+    const possessiveMatch = lower.match(/^(\w+)'s (\w+)/);
+    if (possessiveMatch) {
+      ids.push(possessiveMatch[1]);
+      ids.push(possessiveMatch[1] + 's');
+      ids.push(possessiveMatch[2]);
+      // Combined (e.g., "gemcutters-prism")
+      ids.push(this.slugify(`${possessiveMatch[1]}s ${possessiveMatch[2]}`));
+    }
+
+    // === SPECIAL ITEM MAPPINGS ===
+    const specialMappings: Record<string, string[]> = {
+      // Currency
+      "gemcutter's prism": ['gcp', 'gemcutters-prism'],
+      "glassblower's bauble": ['bauble', 'glassblowers-bauble'],
+      "armourer's scrap": ['scrap', 'armourers-scrap'],
+      "blacksmith's whetstone": ['whetstone', 'blacksmiths-whetstone'],
+      "scroll of wisdom": ['wisdom', 'scroll-of-wisdom'],
+      "arcanist's etcher": ['etcher', 'arcanists-etcher'],
+      "vaal orb": ['vaal'],
+      "mirror of kalandra": ['mirror', 'mirror-of-kalandra'],
+      "regal orb": ['regal'],
+      "ancient orb": ['ancient'],
+      "fracturing orb": ['fracturing-orb'],
+      "hinekora's lock": ['hinekoras-lock'],
+      "crystallised corruption": ['crystallised-corruption'],
+      "vaal cultivation orb": ['vaal-cultivation-orb'],
+      "perfect chaos orb": ['perfect-chaos-orb'],
+      "perfect exalted orb": ['perfect-exalted-orb'],
+      "perfect jeweller's orb": ['perfect-jewellers-orb'],
+      "greater jeweller's orb": ['greater-jewellers-orb'],
+      "lesser jeweller's orb": ['lesser-jewellers-orb'],
+      "greater regal orb": ['greater-regal-orb'],
+      "lesser regal orb": ['lesser-regal-orb'],
+      "core destabiliser": ['core-destabiliser'],
+      "architect's orb": ['architects-orb'],
+      "ancient infuser": ['ancient-infuser'],
+
+      // Fragments
+      "rite of passage": ['rite-of-passage'],
+      "against the darkness": ['against-the-darkness'],
+      "the trialmaster's reliquary key": ['the-trialmasters-reliquary-key'],
+      "tangmazu's reliquary key": ['tangmazus-reliquary-key'],
+      "olroth's reliquary key": ['olroths-reliquary-key'],
+
+      // Expedition
+      "black scythe artifact": ['black-scythe-artifact'],
+      "exotic coinage": ['exotic-coinage'],
+      "sun artifact": ['sun-artifact'],
+      "broken circle artifact": ['broken-circle-artifact'],
+      "order artifact": ['order-artifact'],
+    };
+
+    if (specialMappings[lower]) {
+      ids.push(...specialMappings[lower]);
+    }
+
+    // === GENERIC PATTERNS ===
+
+    // Hyphenated slug of full name
+    ids.push(this.slugify(name));
+
+    // First word if 3+ chars
+    const firstWord = lower.split(/[\s']+/)[0].replace(/[^a-z]/g, '');
+    if (firstWord.length >= 3) {
+      ids.push(firstWord);
+    }
+
+    // Remove duplicates
+    return [...new Set(ids)];
+  }
+
+  /**
+   * Convert name to URL slug
+   */
+  private slugify(name: string): string {
+    return name
+      .toLowerCase()
+      .replace(/'/g, '')
+      .replace(/[^a-z0-9]+/g, '-')
+      .replace(/^-|-$/g, '');
+  }
+
+  /**
+   * Format an ID as a readable name (fallback)
+   */
+  private formatIdAsName(id: string): string {
+    return id
+      .split('-')
+      .map(word => word.charAt(0).toUpperCase() + word.slice(1))
+      .join(' ');
+  }
+}
+
+// Singleton instance
+export const itemNameMapper = new ItemNameMapper();