boki/poe2-data
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Initial commit

Browse source
This commit is contained in:
Boki 2026-02-05 13:52:07 -05:00
commit 84d38c5173
46 changed files with 6819 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

10
packages/database/drizzle.config.ts Normal file
View file

@ -0,0 +1,10 @@
import { defineConfig } from 'drizzle-kit';
export default defineConfig({
schema: './src/schema/index.ts',
out: './drizzle',
dialect: 'sqlite',
dbCredentials: {
url: `file:${process.env.DATABASE_PATH || '../../data/poe2data.db'}`,
},
});

66
packages/database/drizzle/0000_perpetual_riptide.sql Normal file
View file

@ -0,0 +1,66 @@
CREATE TABLE `daily_prices` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`item_id` integer NOT NULL,
`date` text NOT NULL,
`open_value` real,
`close_value` real,
`high_value` real,
`low_value` real,
`avg_volume` real,
`chaos_rate` real,
FOREIGN KEY (`item_id`) REFERENCES `items`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
CREATE UNIQUE INDEX `idx_daily_prices_item_date` ON `daily_prices` (`item_id`,`date`);--> statement-breakpoint
CREATE TABLE `items` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`external_id` text NOT NULL,
`details_id` text NOT NULL,
`league_id` integer NOT NULL,
`category` text NOT NULL,
`name` text NOT NULL,
`icon_url` text,
`created_at` integer,
FOREIGN KEY (`league_id`) REFERENCES `leagues`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
CREATE INDEX `idx_items_league_category` ON `items` (`league_id`,`category`);--> statement-breakpoint
CREATE UNIQUE INDEX `idx_items_external_league` ON `items` (`external_id`,`league_id`);--> statement-breakpoint
CREATE TABLE `leagues` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`name` text NOT NULL,
`display_name` text NOT NULL,
`start_date` integer,
`end_date` integer,
`is_active` integer DEFAULT true,
`created_at` integer
);
--> statement-breakpoint
CREATE UNIQUE INDEX `leagues_name_unique` ON `leagues` (`name`);--> statement-breakpoint
CREATE TABLE `price_history` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`item_id` integer NOT NULL,
`snapshot_id` integer,
`divine_value` real,
`volume` real,
`change_7d` real,
`sparkline_data` text,
`exalted_rate` real,
`chaos_rate` real,
`recorded_at` integer NOT NULL,
FOREIGN KEY (`item_id`) REFERENCES `items`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`snapshot_id`) REFERENCES `snapshots`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
CREATE INDEX `idx_price_history_item_recorded` ON `price_history` (`item_id`,`recorded_at`);--> statement-breakpoint
CREATE INDEX `idx_price_history_snapshot` ON `price_history` (`snapshot_id`);--> statement-breakpoint
CREATE TABLE `snapshots` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`league_id` integer NOT NULL,
`category` text NOT NULL,
`scraped_at` integer NOT NULL,
`status` text DEFAULT 'pending',
`item_count` integer DEFAULT 0,
`error_message` text,
FOREIGN KEY (`league_id`) REFERENCES `leagues`(`id`) ON UPDATE no action ON DELETE no action
);

477
packages/database/drizzle/meta/0000_snapshot.json Normal file
View file

@ -0,0 +1,477 @@
{
"version": "6",
"dialect": "sqlite",
"id": "dc4861e0-d4dd-4c49-bcb1-fd4d5c4da8d4",
"prevId": "00000000-0000-0000-0000-000000000000",
"tables": {
"daily_prices": {
"name": "daily_prices",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"item_id": {
"name": "item_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"date": {
"name": "date",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"open_value": {
"name": "open_value",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"close_value": {
"name": "close_value",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"high_value": {
"name": "high_value",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"low_value": {
"name": "low_value",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"avg_volume": {
"name": "avg_volume",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"chaos_rate": {
"name": "chaos_rate",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {
"idx_daily_prices_item_date": {
"name": "idx_daily_prices_item_date",
"columns": [
"item_id",
"date"
],
"isUnique": true
}
},
"foreignKeys": {
"daily_prices_item_id_items_id_fk": {
"name": "daily_prices_item_id_items_id_fk",
"tableFrom": "daily_prices",
"tableTo": "items",
"columnsFrom": [
"item_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"items": {
"name": "items",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"external_id": {
"name": "external_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"details_id": {
"name": "details_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"league_id": {
"name": "league_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"category": {
"name": "category",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"icon_url": {
"name": "icon_url",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {
"idx_items_league_category": {
"name": "idx_items_league_category",
"columns": [
"league_id",
"category"
],
"isUnique": false
},
"idx_items_external_league": {
"name": "idx_items_external_league",
"columns": [
"external_id",
"league_id"
],
"isUnique": true
}
},
"foreignKeys": {
"items_league_id_leagues_id_fk": {
"name": "items_league_id_leagues_id_fk",
"tableFrom": "items",
"tableTo": "leagues",
"columnsFrom": [
"league_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"leagues": {
"name": "leagues",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"display_name": {
"name": "display_name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"start_date": {
"name": "start_date",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"end_date": {
"name": "end_date",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"is_active": {
"name": "is_active",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": true
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {
"leagues_name_unique": {
"name": "leagues_name_unique",
"columns": [
"name"
],
"isUnique": true
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"price_history": {
"name": "price_history",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"item_id": {
"name": "item_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"snapshot_id": {
"name": "snapshot_id",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"divine_value": {
"name": "divine_value",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"volume": {
"name": "volume",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"change_7d": {
"name": "change_7d",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"sparkline_data": {
"name": "sparkline_data",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"exalted_rate": {
"name": "exalted_rate",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"chaos_rate": {
"name": "chaos_rate",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"recorded_at": {
"name": "recorded_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"idx_price_history_item_recorded": {
"name": "idx_price_history_item_recorded",
"columns": [
"item_id",
"recorded_at"
],
"isUnique": false
},
"idx_price_history_snapshot": {
"name": "idx_price_history_snapshot",
"columns": [
"snapshot_id"
],
"isUnique": false
}
},
"foreignKeys": {
"price_history_item_id_items_id_fk": {
"name": "price_history_item_id_items_id_fk",
"tableFrom": "price_history",
"tableTo": "items",
"columnsFrom": [
"item_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
},
"price_history_snapshot_id_snapshots_id_fk": {
"name": "price_history_snapshot_id_snapshots_id_fk",
"tableFrom": "price_history",
"tableTo": "snapshots",
"columnsFrom": [
"snapshot_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"snapshots": {
"name": "snapshots",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"league_id": {
"name": "league_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"category": {
"name": "category",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"scraped_at": {
"name": "scraped_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": "'pending'"
},
"item_count": {
"name": "item_count",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": 0
},
"error_message": {
"name": "error_message",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {
"snapshots_league_id_leagues_id_fk": {
"name": "snapshots_league_id_leagues_id_fk",
"tableFrom": "snapshots",
"tableTo": "leagues",
"columnsFrom": [
"league_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
}
},
"views": {},
"enums": {},
"_meta": {
"schemas": {},
"tables": {},
"columns": {}
},
"internal": {
"indexes": {}
}
}

13
packages/database/drizzle/meta/_journal.json Normal file
View file

@ -0,0 +1,13 @@
{
"version": "7",
"dialect": "sqlite",
"entries": [
{
"idx": 0,
"version": "6",
"when": 1770307380064,
"tag": "0000_perpetual_riptide",
"breakpoints": true
}
]
}

32
packages/database/package.json Normal file
View file

@ -0,0 +1,32 @@
{
"name": "@poe2data/database",
"version": "1.0.0",
"type": "module",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
"exports": {
".": {
"import": "./dist/index.js",
"types": "./dist/index.d.ts"
}
},
"scripts": {
"build": "tsc",
"dev": "tsc --watch",
"migrate": "tsx src/migrate.ts",
"migrate:drizzle": "drizzle-kit migrate",
"generate": "drizzle-kit generate",
"studio": "drizzle-kit studio"
},
"dependencies": {
"@poe2data/shared": "workspace:*",
"@libsql/client": "^0.14.0",
"drizzle-orm": "^0.38.0"
},
"devDependencies": {
"@types/node": "^20.0.0",
"drizzle-kit": "^0.30.0",
"tsx": "^4.7.0",
"typescript": "^5.3.0"
}
}

39
packages/database/src/index.ts Normal file
View file

@ -0,0 +1,39 @@
import { createClient } from '@libsql/client';
import { drizzle } from 'drizzle-orm/libsql';
import * as schema from './schema/index.js';
import path from 'path';
import { fileURLToPath } from 'url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
// Default database path
const DEFAULT_DB_PATH = path.resolve(__dirname, '../../..', 'data', 'poe2data.db');
let dbInstance: ReturnType<typeof drizzle> | null = null;
let clientInstance: ReturnType<typeof createClient> | null = null;
export function getDatabase(dbPath?: string) {
if (!dbInstance) {
const finalPath = dbPath || process.env.DATABASE_PATH || DEFAULT_DB_PATH;
console.log(`Opening database at: ${finalPath}`);
clientInstance = createClient({
url: `file:${finalPath}`,
});
dbInstance = drizzle(clientInstance, { schema });
}
return dbInstance;
}
export async function closeDatabase() {
if (clientInstance) {
clientInstance.close();
clientInstance = null;
dbInstance = null;
}
}
// Re-export schema and types
export * from './schema/index.js';
export { schema };

42
packages/database/src/migrate.ts Normal file
View file

@ -0,0 +1,42 @@
import { createClient } from '@libsql/client';
import { drizzle } from 'drizzle-orm/libsql';
import { migrate } from 'drizzle-orm/libsql/migrator';
import path from 'path';
import { fileURLToPath } from 'url';
import fs from 'fs';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const DB_PATH = process.env.DATABASE_PATH || path.resolve(__dirname, '../../..', 'data', 'poe2data.db');
const MIGRATIONS_PATH = path.resolve(__dirname, '..', 'drizzle');
async function main() {
console.log('Running database migrations...');
console.log(`Database path: ${DB_PATH}`);
console.log(`Migrations path: ${MIGRATIONS_PATH}`);
// Ensure data directory exists
const dataDir = path.dirname(DB_PATH);
if (!fs.existsSync(dataDir)) {
fs.mkdirSync(dataDir, { recursive: true });
console.log(`Created data directory: ${dataDir}`);
}
const client = createClient({
url: `file:${DB_PATH}`,
});
const db = drizzle(client);
try {
await migrate(db, { migrationsFolder: MIGRATIONS_PATH });
console.log('Migrations completed successfully!');
} catch (error) {
console.error('Migration failed:', error);
process.exit(1);
} finally {
client.close();
}
}
main();

95
packages/database/src/schema/index.ts Normal file
View file

@ -0,0 +1,95 @@
import { sqliteTable, text, integer, real, index, uniqueIndex } from 'drizzle-orm/sqlite-core';
// Leagues table
export const leagues = sqliteTable('leagues', {
id: integer('id').primaryKey({ autoIncrement: true }),
name: text('name').notNull().unique(),
displayName: text('display_name').notNull(),
startDate: integer('start_date', { mode: 'timestamp' }),
endDate: integer('end_date', { mode: 'timestamp' }),
isActive: integer('is_active', { mode: 'boolean' }).default(true),
createdAt: integer('created_at', { mode: 'timestamp' }).$defaultFn(() => new Date()),
});
// Items table - stores item metadata
export const items = sqliteTable('items', {
id: integer('id').primaryKey({ autoIncrement: true }),
externalId: text('external_id').notNull(), // poe.ninja ID (e.g., "divine", "chaos")
detailsId: text('details_id').notNull(), // poe.ninja details ID (e.g., "divine-orb")
leagueId: integer('league_id').references(() => leagues.id).notNull(),
category: text('category').notNull(), // Currency, Fragments, etc.
name: text('name').notNull(),
iconUrl: text('icon_url'),
createdAt: integer('created_at', { mode: 'timestamp' }).$defaultFn(() => new Date()),
}, (table) => [
index('idx_items_league_category').on(table.leagueId, table.category),
uniqueIndex('idx_items_external_league').on(table.externalId, table.leagueId),
]);
// Snapshots table - tracks when we scraped data
export const snapshots = sqliteTable('snapshots', {
id: integer('id').primaryKey({ autoIncrement: true }),
leagueId: integer('league_id').references(() => leagues.id).notNull(),
category: text('category').notNull(),
scrapedAt: integer('scraped_at', { mode: 'timestamp' }).notNull().$defaultFn(() => new Date()),
status: text('status', { enum: ['pending', 'success', 'failed'] }).default('pending'),
itemCount: integer('item_count').default(0),
errorMessage: text('error_message'),
});
// Price history table - stores historical prices
export const priceHistory = sqliteTable('price_history', {
id: integer('id').primaryKey({ autoIncrement: true }),
itemId: integer('item_id').references(() => items.id).notNull(),
snapshotId: integer('snapshot_id').references(() => snapshots.id),
// Value in divines (primary currency)
divineValue: real('divine_value'),
// Trading volume
volume: real('volume'),
// 7-day change percentage
change7d: real('change_7d'),
// Sparkline data (JSON array of 7 values)
sparklineData: text('sparkline_data'),
// Exchange rates at time of snapshot
exaltedRate: real('exalted_rate'), // exalted per divine
chaosRate: real('chaos_rate'), // chaos per divine
recordedAt: integer('recorded_at', { mode: 'timestamp' }).notNull().$defaultFn(() => new Date()),
}, (table) => [
index('idx_price_history_item_recorded').on(table.itemId, table.recordedAt),
index('idx_price_history_snapshot').on(table.snapshotId),
]);
// Daily price aggregates - for faster trend queries
export const dailyPrices = sqliteTable('daily_prices', {
id: integer('id').primaryKey({ autoIncrement: true }),
itemId: integer('item_id').references(() => items.id).notNull(),
date: text('date').notNull(), // YYYY-MM-DD format
openValue: real('open_value'),
closeValue: real('close_value'),
highValue: real('high_value'),
lowValue: real('low_value'),
avgVolume: real('avg_volume'),
chaosRate: real('chaos_rate'), // chaos per divine for that day
}, (table) => [
uniqueIndex('idx_daily_prices_item_date').on(table.itemId, table.date),
]);
// Type exports for use in other packages
export type League = typeof leagues.$inferSelect;
export type NewLeague = typeof leagues.$inferInsert;
export type Item = typeof items.$inferSelect;
export type NewItem = typeof items.$inferInsert;
export type Snapshot = typeof snapshots.$inferSelect;
export type NewSnapshot = typeof snapshots.$inferInsert;
export type PriceHistory = typeof priceHistory.$inferSelect;
export type NewPriceHistory = typeof priceHistory.$inferInsert;
export type DailyPrice = typeof dailyPrices.$inferSelect;
export type NewDailyPrice = typeof dailyPrices.$inferInsert;

8
packages/database/tsconfig.json Normal file
View file

@ -0,0 +1,8 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"outDir": "dist",
"rootDir": "src"
},
"include": ["src/**/*"]
}