Initial commit
This commit is contained in:
commit
84d38c5173
46 changed files with 6819 additions and 0 deletions
29
.claude/settings.local.json
Normal file
29
.claude/settings.local.json
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
{
|
||||||
|
"permissions": {
|
||||||
|
"allow": [
|
||||||
|
"WebSearch",
|
||||||
|
"WebFetch(domain:www.pathofexile.com)",
|
||||||
|
"WebFetch(domain:poe.ninja)",
|
||||||
|
"WebFetch(domain:github.com)",
|
||||||
|
"WebFetch(domain:raw.githubusercontent.com)",
|
||||||
|
"Bash(pnpm install:*)",
|
||||||
|
"Bash(npm install:*)",
|
||||||
|
"Bash(pnpm approve-builds:*)",
|
||||||
|
"Bash(npx tsx:*)",
|
||||||
|
"Bash(npx tsc)",
|
||||||
|
"Bash(curl:*)",
|
||||||
|
"Bash(pnpm rebuild:*)",
|
||||||
|
"Bash(npx drizzle-kit generate)",
|
||||||
|
"Bash(npm rebuild:*)",
|
||||||
|
"Bash(pnpm migrate:*)",
|
||||||
|
"Bash(taskkill //F //FI \"WINDOWTITLE eq poe2data*\")",
|
||||||
|
"Bash(pkill:*)",
|
||||||
|
"Bash(taskkill:*)",
|
||||||
|
"Bash(timeout:*)",
|
||||||
|
"Bash(head -c 2000 curl -s \"https://poe.ninja/poe2/economy\")",
|
||||||
|
"Bash(head -5 curl -s \"https://poe.ninja/poe2/api/economy/exchange/current/search?league=Standard\")",
|
||||||
|
"Bash(pnpm scraper:*)",
|
||||||
|
"Bash(pnpm db:migrate:*)"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
8
.gitignore
vendored
Normal file
8
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
node_modules/
|
||||||
|
dist/
|
||||||
|
*.db
|
||||||
|
*.db-journal
|
||||||
|
.env
|
||||||
|
.env.local
|
||||||
|
.DS_Store
|
||||||
|
*.log
|
||||||
13
apps/dashboard/index.html
Normal file
13
apps/dashboard/index.html
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8" />
|
||||||
|
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<title>POE2 Data Dashboard</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div id="root"></div>
|
||||||
|
<script type="module" src="/src/main.tsx"></script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
27
apps/dashboard/package.json
Normal file
27
apps/dashboard/package.json
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
{
|
||||||
|
"name": "@poe2data/dashboard",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"dev": "vite",
|
||||||
|
"build": "tsc && vite build",
|
||||||
|
"preview": "vite preview"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"react": "^18.2.0",
|
||||||
|
"react-dom": "^18.2.0",
|
||||||
|
"react-router-dom": "^6.22.0",
|
||||||
|
"recharts": "^2.12.0",
|
||||||
|
"@tanstack/react-query": "^5.17.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/react": "^18.2.0",
|
||||||
|
"@types/react-dom": "^18.2.0",
|
||||||
|
"@vitejs/plugin-react": "^4.2.0",
|
||||||
|
"autoprefixer": "^10.4.17",
|
||||||
|
"postcss": "^8.4.35",
|
||||||
|
"tailwindcss": "^3.4.1",
|
||||||
|
"typescript": "^5.3.0",
|
||||||
|
"vite": "^5.1.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
6
apps/dashboard/postcss.config.js
Normal file
6
apps/dashboard/postcss.config.js
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
export default {
|
||||||
|
plugins: {
|
||||||
|
tailwindcss: {},
|
||||||
|
autoprefixer: {},
|
||||||
|
},
|
||||||
|
};
|
||||||
53
apps/dashboard/src/App.tsx
Normal file
53
apps/dashboard/src/App.tsx
Normal file
|
|
@ -0,0 +1,53 @@
|
||||||
|
import { Routes, Route, Link, useLocation } from 'react-router-dom';
|
||||||
|
import Dashboard from './pages/Dashboard';
|
||||||
|
import Items from './pages/Items';
|
||||||
|
import LeagueStart from './pages/LeagueStart';
|
||||||
|
|
||||||
|
const navItems = [
|
||||||
|
{ path: '/', label: 'Dashboard' },
|
||||||
|
{ path: '/items', label: 'Items' },
|
||||||
|
{ path: '/league-start', label: 'League Start' },
|
||||||
|
];
|
||||||
|
|
||||||
|
function App() {
|
||||||
|
const location = useLocation();
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="min-h-screen">
|
||||||
|
{/* Header */}
|
||||||
|
<header className="bg-gray-800 border-b border-gray-700">
|
||||||
|
<div className="max-w-7xl mx-auto px-4 py-4">
|
||||||
|
<div className="flex items-center justify-between">
|
||||||
|
<h1 className="text-2xl font-bold text-amber-500">POE2 Data</h1>
|
||||||
|
<nav className="flex gap-4">
|
||||||
|
{navItems.map((item) => (
|
||||||
|
<Link
|
||||||
|
key={item.path}
|
||||||
|
to={item.path}
|
||||||
|
className={`px-3 py-2 rounded-md text-sm font-medium transition-colors ${
|
||||||
|
location.pathname === item.path
|
||||||
|
? 'bg-amber-600 text-white'
|
||||||
|
: 'text-gray-300 hover:bg-gray-700'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
{item.label}
|
||||||
|
</Link>
|
||||||
|
))}
|
||||||
|
</nav>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
{/* Main Content */}
|
||||||
|
<main className="max-w-7xl mx-auto px-4 py-6">
|
||||||
|
<Routes>
|
||||||
|
<Route path="/" element={<Dashboard />} />
|
||||||
|
<Route path="/items" element={<Items />} />
|
||||||
|
<Route path="/league-start" element={<LeagueStart />} />
|
||||||
|
</Routes>
|
||||||
|
</main>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default App;
|
||||||
7
apps/dashboard/src/index.css
Normal file
7
apps/dashboard/src/index.css
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
@tailwind base;
|
||||||
|
@tailwind components;
|
||||||
|
@tailwind utilities;
|
||||||
|
|
||||||
|
body {
|
||||||
|
@apply bg-gray-900 text-gray-100;
|
||||||
|
}
|
||||||
61
apps/dashboard/src/lib/api.ts
Normal file
61
apps/dashboard/src/lib/api.ts
Normal file
|
|
@ -0,0 +1,61 @@
|
||||||
|
const API_BASE = '/api/v1';
|
||||||
|
|
||||||
|
export interface Item {
|
||||||
|
id: number;
|
||||||
|
name: string;
|
||||||
|
category: string;
|
||||||
|
iconUrl: string | null;
|
||||||
|
currentValue: number;
|
||||||
|
change7d: number;
|
||||||
|
volume: number;
|
||||||
|
chaosValue: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CategorySummary {
|
||||||
|
category: string;
|
||||||
|
itemCount: number;
|
||||||
|
topItem: string;
|
||||||
|
topValue: number;
|
||||||
|
avgChange7d: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LeagueStartPriority {
|
||||||
|
name: string;
|
||||||
|
category: string;
|
||||||
|
currentValue: number;
|
||||||
|
change7d: number;
|
||||||
|
volume: number;
|
||||||
|
priority: 'high' | 'medium' | 'low';
|
||||||
|
reason: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchItems(category?: string): Promise<{ items: Item[]; count: number }> {
|
||||||
|
const url = category ? `${API_BASE}/items?category=${category}` : `${API_BASE}/items`;
|
||||||
|
const res = await fetch(url);
|
||||||
|
return res.json();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchMovers(): Promise<{ gainers: Item[]; losers: Item[] }> {
|
||||||
|
const res = await fetch(`${API_BASE}/trends/movers`);
|
||||||
|
return res.json();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchCategories(): Promise<{ categories: CategorySummary[] }> {
|
||||||
|
const res = await fetch(`${API_BASE}/categories`);
|
||||||
|
return res.json();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchLeagueStartPriorities(): Promise<{
|
||||||
|
priorities: LeagueStartPriority[];
|
||||||
|
summary: { high: number; medium: number; low: number };
|
||||||
|
}> {
|
||||||
|
const res = await fetch(`${API_BASE}/league-start/priorities`);
|
||||||
|
return res.json();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchItemHistory(id: number): Promise<{
|
||||||
|
history: { timestamp: number; value: number; volume: number }[];
|
||||||
|
}> {
|
||||||
|
const res = await fetch(`${API_BASE}/items/${id}/history`);
|
||||||
|
return res.json();
|
||||||
|
}
|
||||||
25
apps/dashboard/src/main.tsx
Normal file
25
apps/dashboard/src/main.tsx
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
import React from 'react';
|
||||||
|
import ReactDOM from 'react-dom/client';
|
||||||
|
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
|
||||||
|
import { BrowserRouter } from 'react-router-dom';
|
||||||
|
import App from './App';
|
||||||
|
import './index.css';
|
||||||
|
|
||||||
|
const queryClient = new QueryClient({
|
||||||
|
defaultOptions: {
|
||||||
|
queries: {
|
||||||
|
staleTime: 1000 * 60 * 5, // 5 minutes
|
||||||
|
refetchInterval: 1000 * 60 * 5, // Refetch every 5 minutes
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
ReactDOM.createRoot(document.getElementById('root')!).render(
|
||||||
|
<React.StrictMode>
|
||||||
|
<QueryClientProvider client={queryClient}>
|
||||||
|
<BrowserRouter>
|
||||||
|
<App />
|
||||||
|
</BrowserRouter>
|
||||||
|
</QueryClientProvider>
|
||||||
|
</React.StrictMode>
|
||||||
|
);
|
||||||
156
apps/dashboard/src/pages/Dashboard.tsx
Normal file
156
apps/dashboard/src/pages/Dashboard.tsx
Normal file
|
|
@ -0,0 +1,156 @@
|
||||||
|
import { useQuery } from '@tanstack/react-query';
|
||||||
|
import { fetchMovers, fetchCategories } from '../lib/api';
|
||||||
|
import { BarChart, Bar, XAxis, YAxis, Tooltip, ResponsiveContainer, Cell } from 'recharts';
|
||||||
|
|
||||||
|
function formatValue(value: number): string {
|
||||||
|
if (value >= 1000) return `${(value / 1000).toFixed(1)}k`;
|
||||||
|
if (value >= 1) return value.toFixed(2);
|
||||||
|
return value.toFixed(4);
|
||||||
|
}
|
||||||
|
|
||||||
|
function ChangeIndicator({ change }: { change: number }) {
|
||||||
|
const color = change >= 0 ? 'text-green-400' : 'text-red-400';
|
||||||
|
const arrow = change >= 0 ? '↑' : '↓';
|
||||||
|
return (
|
||||||
|
<span className={color}>
|
||||||
|
{arrow} {Math.abs(change).toFixed(1)}%
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function Dashboard() {
|
||||||
|
const { data: movers, isLoading: moversLoading } = useQuery({
|
||||||
|
queryKey: ['movers'],
|
||||||
|
queryFn: fetchMovers,
|
||||||
|
});
|
||||||
|
|
||||||
|
const { data: categories, isLoading: categoriesLoading } = useQuery({
|
||||||
|
queryKey: ['categories'],
|
||||||
|
queryFn: fetchCategories,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (moversLoading || categoriesLoading) {
|
||||||
|
return <div className="text-center py-10">Loading...</div>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const chartData = movers?.gainers.slice(0, 10).map((item) => ({
|
||||||
|
name: item.name.length > 15 ? item.name.substring(0, 15) + '...' : item.name,
|
||||||
|
change: item.change7d,
|
||||||
|
}));
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-8">
|
||||||
|
{/* Categories Overview */}
|
||||||
|
<section>
|
||||||
|
<h2 className="text-xl font-semibold mb-4">Categories Overview</h2>
|
||||||
|
<div className="grid grid-cols-2 md:grid-cols-4 lg:grid-cols-5 gap-4">
|
||||||
|
{categories?.categories.map((cat) => (
|
||||||
|
<div key={cat.category} className="bg-gray-800 rounded-lg p-4">
|
||||||
|
<div className="text-sm text-gray-400">{cat.category}</div>
|
||||||
|
<div className="text-lg font-semibold">{cat.itemCount} items</div>
|
||||||
|
<div className="text-sm text-amber-500">{cat.topItem}</div>
|
||||||
|
<div className="text-xs text-gray-500">
|
||||||
|
Top: {formatValue(cat.topValue)} div
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{/* Top Gainers Chart */}
|
||||||
|
<section>
|
||||||
|
<h2 className="text-xl font-semibold mb-4">Top Gainers (7d)</h2>
|
||||||
|
<div className="bg-gray-800 rounded-lg p-4 h-64">
|
||||||
|
<ResponsiveContainer width="100%" height="100%">
|
||||||
|
<BarChart data={chartData} layout="vertical">
|
||||||
|
<XAxis type="number" tickFormatter={(v) => `${v}%`} />
|
||||||
|
<YAxis type="category" dataKey="name" width={120} tick={{ fontSize: 12 }} />
|
||||||
|
<Tooltip
|
||||||
|
formatter={(value: number) => [`${value.toFixed(1)}%`, '7d Change']}
|
||||||
|
contentStyle={{ backgroundColor: '#1f2937', border: 'none' }}
|
||||||
|
/>
|
||||||
|
<Bar dataKey="change" radius={[0, 4, 4, 0]}>
|
||||||
|
{chartData?.map((entry, index) => (
|
||||||
|
<Cell key={index} fill={entry.change >= 0 ? '#10b981' : '#ef4444'} />
|
||||||
|
))}
|
||||||
|
</Bar>
|
||||||
|
</BarChart>
|
||||||
|
</ResponsiveContainer>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{/* Top Movers Tables */}
|
||||||
|
<div className="grid md:grid-cols-2 gap-6">
|
||||||
|
{/* Gainers */}
|
||||||
|
<section>
|
||||||
|
<h2 className="text-xl font-semibold mb-4 text-green-400">Top Gainers</h2>
|
||||||
|
<div className="bg-gray-800 rounded-lg overflow-hidden">
|
||||||
|
<table className="w-full">
|
||||||
|
<thead className="bg-gray-700">
|
||||||
|
<tr>
|
||||||
|
<th className="px-4 py-2 text-left text-sm">Item</th>
|
||||||
|
<th className="px-4 py-2 text-right text-sm">Value</th>
|
||||||
|
<th className="px-4 py-2 text-right text-sm">7d</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{movers?.gainers.slice(0, 10).map((item) => (
|
||||||
|
<tr key={item.id} className="border-t border-gray-700 hover:bg-gray-750">
|
||||||
|
<td className="px-4 py-2">
|
||||||
|
<div className="font-medium">{item.name}</div>
|
||||||
|
<div className="text-xs text-gray-400">{item.category}</div>
|
||||||
|
</td>
|
||||||
|
<td className="px-4 py-2 text-right">
|
||||||
|
<div>{formatValue(item.currentValue)} div</div>
|
||||||
|
<div className="text-xs text-gray-400">
|
||||||
|
{formatValue(item.chaosValue)} chaos
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
<td className="px-4 py-2 text-right">
|
||||||
|
<ChangeIndicator change={item.change7d} />
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{/* Losers */}
|
||||||
|
<section>
|
||||||
|
<h2 className="text-xl font-semibold mb-4 text-red-400">Top Losers</h2>
|
||||||
|
<div className="bg-gray-800 rounded-lg overflow-hidden">
|
||||||
|
<table className="w-full">
|
||||||
|
<thead className="bg-gray-700">
|
||||||
|
<tr>
|
||||||
|
<th className="px-4 py-2 text-left text-sm">Item</th>
|
||||||
|
<th className="px-4 py-2 text-right text-sm">Value</th>
|
||||||
|
<th className="px-4 py-2 text-right text-sm">7d</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{movers?.losers.slice(0, 10).map((item) => (
|
||||||
|
<tr key={item.id} className="border-t border-gray-700 hover:bg-gray-750">
|
||||||
|
<td className="px-4 py-2">
|
||||||
|
<div className="font-medium">{item.name}</div>
|
||||||
|
<div className="text-xs text-gray-400">{item.category}</div>
|
||||||
|
</td>
|
||||||
|
<td className="px-4 py-2 text-right">
|
||||||
|
<div>{formatValue(item.currentValue)} div</div>
|
||||||
|
<div className="text-xs text-gray-400">
|
||||||
|
{formatValue(item.chaosValue)} chaos
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
<td className="px-4 py-2 text-right">
|
||||||
|
<ChangeIndicator change={item.change7d} />
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
156
apps/dashboard/src/pages/Items.tsx
Normal file
156
apps/dashboard/src/pages/Items.tsx
Normal file
|
|
@ -0,0 +1,156 @@
|
||||||
|
import { useState } from 'react';
|
||||||
|
import { useQuery } from '@tanstack/react-query';
|
||||||
|
import { fetchItems, fetchCategories, Item } from '../lib/api';
|
||||||
|
|
||||||
|
function formatValue(value: number): string {
|
||||||
|
if (value >= 1000) return `${(value / 1000).toFixed(1)}k`;
|
||||||
|
if (value >= 1) return value.toFixed(2);
|
||||||
|
return value.toFixed(4);
|
||||||
|
}
|
||||||
|
|
||||||
|
function ChangeIndicator({ change }: { change: number }) {
|
||||||
|
const color = change >= 0 ? 'text-green-400' : 'text-red-400';
|
||||||
|
const arrow = change >= 0 ? '↑' : '↓';
|
||||||
|
return (
|
||||||
|
<span className={color}>
|
||||||
|
{arrow} {Math.abs(change).toFixed(1)}%
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function Items() {
|
||||||
|
const [selectedCategory, setSelectedCategory] = useState<string>('');
|
||||||
|
const [searchTerm, setSearchTerm] = useState('');
|
||||||
|
const [sortBy, setSortBy] = useState<'value' | 'change' | 'volume'>('value');
|
||||||
|
|
||||||
|
const { data: categories } = useQuery({
|
||||||
|
queryKey: ['categories'],
|
||||||
|
queryFn: fetchCategories,
|
||||||
|
});
|
||||||
|
|
||||||
|
const { data, isLoading } = useQuery({
|
||||||
|
queryKey: ['items', selectedCategory],
|
||||||
|
queryFn: () => fetchItems(selectedCategory || undefined),
|
||||||
|
});
|
||||||
|
|
||||||
|
let filteredItems = data?.items || [];
|
||||||
|
|
||||||
|
// Filter by search
|
||||||
|
if (searchTerm) {
|
||||||
|
filteredItems = filteredItems.filter((item) =>
|
||||||
|
item.name.toLowerCase().includes(searchTerm.toLowerCase())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort
|
||||||
|
filteredItems = [...filteredItems].sort((a, b) => {
|
||||||
|
if (sortBy === 'value') return b.currentValue - a.currentValue;
|
||||||
|
if (sortBy === 'change') return b.change7d - a.change7d;
|
||||||
|
return b.volume - a.volume;
|
||||||
|
});
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-6">
|
||||||
|
{/* Filters */}
|
||||||
|
<div className="flex flex-wrap gap-4 items-center">
|
||||||
|
<div>
|
||||||
|
<label className="block text-sm text-gray-400 mb-1">Category</label>
|
||||||
|
<select
|
||||||
|
value={selectedCategory}
|
||||||
|
onChange={(e) => setSelectedCategory(e.target.value)}
|
||||||
|
className="bg-gray-800 border border-gray-700 rounded px-3 py-2 text-sm"
|
||||||
|
>
|
||||||
|
<option value="">All Categories</option>
|
||||||
|
{categories?.categories.map((cat) => (
|
||||||
|
<option key={cat.category} value={cat.category}>
|
||||||
|
{cat.category} ({cat.itemCount})
|
||||||
|
</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label className="block text-sm text-gray-400 mb-1">Search</label>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
value={searchTerm}
|
||||||
|
onChange={(e) => setSearchTerm(e.target.value)}
|
||||||
|
placeholder="Search items..."
|
||||||
|
className="bg-gray-800 border border-gray-700 rounded px-3 py-2 text-sm w-64"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label className="block text-sm text-gray-400 mb-1">Sort By</label>
|
||||||
|
<select
|
||||||
|
value={sortBy}
|
||||||
|
onChange={(e) => setSortBy(e.target.value as any)}
|
||||||
|
className="bg-gray-800 border border-gray-700 rounded px-3 py-2 text-sm"
|
||||||
|
>
|
||||||
|
<option value="value">Value</option>
|
||||||
|
<option value="change">7d Change</option>
|
||||||
|
<option value="volume">Volume</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="ml-auto text-sm text-gray-400">
|
||||||
|
{filteredItems.length} items
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Items Table */}
|
||||||
|
{isLoading ? (
|
||||||
|
<div className="text-center py-10">Loading...</div>
|
||||||
|
) : (
|
||||||
|
<div className="bg-gray-800 rounded-lg overflow-hidden">
|
||||||
|
<table className="w-full">
|
||||||
|
<thead className="bg-gray-700">
|
||||||
|
<tr>
|
||||||
|
<th className="px-4 py-3 text-left text-sm">Item</th>
|
||||||
|
<th className="px-4 py-3 text-left text-sm">Category</th>
|
||||||
|
<th className="px-4 py-3 text-right text-sm">Value (Divine)</th>
|
||||||
|
<th className="px-4 py-3 text-right text-sm">Value (Chaos)</th>
|
||||||
|
<th className="px-4 py-3 text-right text-sm">7d Change</th>
|
||||||
|
<th className="px-4 py-3 text-right text-sm">Volume</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{filteredItems.map((item) => (
|
||||||
|
<tr
|
||||||
|
key={item.id}
|
||||||
|
className="border-t border-gray-700 hover:bg-gray-750"
|
||||||
|
>
|
||||||
|
<td className="px-4 py-3">
|
||||||
|
<div className="flex items-center gap-3">
|
||||||
|
{item.iconUrl && (
|
||||||
|
<img
|
||||||
|
src={`https://web.poecdn.com${item.iconUrl}`}
|
||||||
|
alt=""
|
||||||
|
className="w-8 h-8 object-contain"
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
<span className="font-medium">{item.name}</span>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
<td className="px-4 py-3 text-gray-400">{item.category}</td>
|
||||||
|
<td className="px-4 py-3 text-right font-mono">
|
||||||
|
{formatValue(item.currentValue)}
|
||||||
|
</td>
|
||||||
|
<td className="px-4 py-3 text-right font-mono text-gray-400">
|
||||||
|
{formatValue(item.chaosValue)}
|
||||||
|
</td>
|
||||||
|
<td className="px-4 py-3 text-right">
|
||||||
|
<ChangeIndicator change={item.change7d} />
|
||||||
|
</td>
|
||||||
|
<td className="px-4 py-3 text-right text-gray-400">
|
||||||
|
{formatValue(item.volume)}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
160
apps/dashboard/src/pages/LeagueStart.tsx
Normal file
160
apps/dashboard/src/pages/LeagueStart.tsx
Normal file
|
|
@ -0,0 +1,160 @@
|
||||||
|
import { useQuery } from '@tanstack/react-query';
|
||||||
|
import { fetchLeagueStartPriorities } from '../lib/api';
|
||||||
|
|
||||||
|
function formatValue(value: number): string {
|
||||||
|
if (value >= 1000) return `${(value / 1000).toFixed(1)}k`;
|
||||||
|
if (value >= 1) return value.toFixed(2);
|
||||||
|
return value.toFixed(4);
|
||||||
|
}
|
||||||
|
|
||||||
|
function ChangeIndicator({ change }: { change: number }) {
|
||||||
|
const color = change >= 0 ? 'text-green-400' : 'text-red-400';
|
||||||
|
const arrow = change >= 0 ? '↑' : '↓';
|
||||||
|
return (
|
||||||
|
<span className={color}>
|
||||||
|
{arrow} {Math.abs(change).toFixed(1)}%
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function PriorityBadge({ priority }: { priority: 'high' | 'medium' | 'low' }) {
|
||||||
|
const colors = {
|
||||||
|
high: 'bg-red-500/20 text-red-400 border-red-500/50',
|
||||||
|
medium: 'bg-yellow-500/20 text-yellow-400 border-yellow-500/50',
|
||||||
|
low: 'bg-gray-500/20 text-gray-400 border-gray-500/50',
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<span className={`px-2 py-1 rounded text-xs font-medium border ${colors[priority]}`}>
|
||||||
|
{priority.toUpperCase()}
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function LeagueStart() {
|
||||||
|
const { data, isLoading } = useQuery({
|
||||||
|
queryKey: ['league-start'],
|
||||||
|
queryFn: fetchLeagueStartPriorities,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (isLoading) {
|
||||||
|
return <div className="text-center py-10">Loading...</div>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const highPriority = data?.priorities.filter((p) => p.priority === 'high') || [];
|
||||||
|
const mediumPriority = data?.priorities.filter((p) => p.priority === 'medium') || [];
|
||||||
|
const lowPriority = data?.priorities.filter((p) => p.priority === 'low') || [];
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-8">
|
||||||
|
{/* Summary Cards */}
|
||||||
|
<div className="grid grid-cols-3 gap-4">
|
||||||
|
<div className="bg-red-500/10 border border-red-500/30 rounded-lg p-4 text-center">
|
||||||
|
<div className="text-3xl font-bold text-red-400">{data?.summary.high}</div>
|
||||||
|
<div className="text-sm text-gray-400">High Priority</div>
|
||||||
|
</div>
|
||||||
|
<div className="bg-yellow-500/10 border border-yellow-500/30 rounded-lg p-4 text-center">
|
||||||
|
<div className="text-3xl font-bold text-yellow-400">{data?.summary.medium}</div>
|
||||||
|
<div className="text-sm text-gray-400">Medium Priority</div>
|
||||||
|
</div>
|
||||||
|
<div className="bg-gray-500/10 border border-gray-500/30 rounded-lg p-4 text-center">
|
||||||
|
<div className="text-3xl font-bold text-gray-400">{data?.summary.low}</div>
|
||||||
|
<div className="text-sm text-gray-400">Low Priority</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* High Priority Section */}
|
||||||
|
<section>
|
||||||
|
<h2 className="text-xl font-semibold mb-4 text-red-400">
|
||||||
|
High Priority - Farm These First
|
||||||
|
</h2>
|
||||||
|
<div className="bg-gray-800 rounded-lg overflow-hidden">
|
||||||
|
<table className="w-full">
|
||||||
|
<thead className="bg-gray-700">
|
||||||
|
<tr>
|
||||||
|
<th className="px-4 py-3 text-left text-sm">Item</th>
|
||||||
|
<th className="px-4 py-3 text-left text-sm">Category</th>
|
||||||
|
<th className="px-4 py-3 text-right text-sm">Value</th>
|
||||||
|
<th className="px-4 py-3 text-right text-sm">7d Change</th>
|
||||||
|
<th className="px-4 py-3 text-left text-sm">Reason</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{highPriority.slice(0, 15).map((item, i) => (
|
||||||
|
<tr key={i} className="border-t border-gray-700 hover:bg-gray-750">
|
||||||
|
<td className="px-4 py-3 font-medium">{item.name}</td>
|
||||||
|
<td className="px-4 py-3 text-gray-400">{item.category}</td>
|
||||||
|
<td className="px-4 py-3 text-right font-mono">
|
||||||
|
{formatValue(item.currentValue)} div
|
||||||
|
</td>
|
||||||
|
<td className="px-4 py-3 text-right">
|
||||||
|
<ChangeIndicator change={item.change7d} />
|
||||||
|
</td>
|
||||||
|
<td className="px-4 py-3 text-sm text-gray-400">{item.reason}</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{/* Medium Priority Section */}
|
||||||
|
<section>
|
||||||
|
<h2 className="text-xl font-semibold mb-4 text-yellow-400">
|
||||||
|
Medium Priority - Good Opportunities
|
||||||
|
</h2>
|
||||||
|
<div className="bg-gray-800 rounded-lg overflow-hidden">
|
||||||
|
<table className="w-full">
|
||||||
|
<thead className="bg-gray-700">
|
||||||
|
<tr>
|
||||||
|
<th className="px-4 py-3 text-left text-sm">Item</th>
|
||||||
|
<th className="px-4 py-3 text-left text-sm">Category</th>
|
||||||
|
<th className="px-4 py-3 text-right text-sm">Value</th>
|
||||||
|
<th className="px-4 py-3 text-right text-sm">7d Change</th>
|
||||||
|
<th className="px-4 py-3 text-left text-sm">Reason</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{mediumPriority.slice(0, 15).map((item, i) => (
|
||||||
|
<tr key={i} className="border-t border-gray-700 hover:bg-gray-750">
|
||||||
|
<td className="px-4 py-3 font-medium">{item.name}</td>
|
||||||
|
<td className="px-4 py-3 text-gray-400">{item.category}</td>
|
||||||
|
<td className="px-4 py-3 text-right font-mono">
|
||||||
|
{formatValue(item.currentValue)} div
|
||||||
|
</td>
|
||||||
|
<td className="px-4 py-3 text-right">
|
||||||
|
<ChangeIndicator change={item.change7d} />
|
||||||
|
</td>
|
||||||
|
<td className="px-4 py-3 text-sm text-gray-400">{item.reason}</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{/* Tips Section */}
|
||||||
|
<section className="bg-amber-500/10 border border-amber-500/30 rounded-lg p-6">
|
||||||
|
<h3 className="text-lg font-semibold text-amber-400 mb-3">League Start Tips</h3>
|
||||||
|
<ul className="space-y-2 text-sm text-gray-300">
|
||||||
|
<li>
|
||||||
|
<strong>High Priority Items:</strong> These have high value and good demand - focus on
|
||||||
|
farming these early in the league when prices are typically higher.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<strong>Rising Prices:</strong> Items with large positive 7d changes may continue
|
||||||
|
rising - consider holding these.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<strong>Volume Matters:</strong> High volume means easy selling - important for league
|
||||||
|
start currency building.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<strong>Check Daily:</strong> Prices change rapidly in the first weeks - scrape data
|
||||||
|
frequently!
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
8
apps/dashboard/tailwind.config.js
Normal file
8
apps/dashboard/tailwind.config.js
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
/** @type {import('tailwindcss').Config} */
|
||||||
|
export default {
|
||||||
|
content: ['./index.html', './src/**/*.{js,ts,jsx,tsx}'],
|
||||||
|
theme: {
|
||||||
|
extend: {},
|
||||||
|
},
|
||||||
|
plugins: [],
|
||||||
|
};
|
||||||
21
apps/dashboard/tsconfig.json
Normal file
21
apps/dashboard/tsconfig.json
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ES2020",
|
||||||
|
"useDefineForClassFields": true,
|
||||||
|
"lib": ["ES2020", "DOM", "DOM.Iterable"],
|
||||||
|
"module": "ESNext",
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"allowImportingTsExtensions": true,
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"isolatedModules": true,
|
||||||
|
"noEmit": true,
|
||||||
|
"jsx": "react-jsx",
|
||||||
|
"strict": true,
|
||||||
|
"noUnusedLocals": true,
|
||||||
|
"noUnusedParameters": true,
|
||||||
|
"noFallthroughCasesInSwitch": true
|
||||||
|
},
|
||||||
|
"include": ["src"],
|
||||||
|
"references": [{ "path": "./tsconfig.node.json" }]
|
||||||
|
}
|
||||||
10
apps/dashboard/tsconfig.node.json
Normal file
10
apps/dashboard/tsconfig.node.json
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"composite": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"allowSyntheticDefaultImports": true
|
||||||
|
},
|
||||||
|
"include": ["vite.config.ts"]
|
||||||
|
}
|
||||||
15
apps/dashboard/vite.config.ts
Normal file
15
apps/dashboard/vite.config.ts
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
import { defineConfig } from 'vite';
|
||||||
|
import react from '@vitejs/plugin-react';
|
||||||
|
|
||||||
|
export default defineConfig({
|
||||||
|
plugins: [react()],
|
||||||
|
server: {
|
||||||
|
port: 5173,
|
||||||
|
proxy: {
|
||||||
|
'/api': {
|
||||||
|
target: 'http://localhost:3001',
|
||||||
|
changeOrigin: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
18
package.json
Normal file
18
package.json
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
{
|
||||||
|
"name": "poe2data",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"private": true,
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"dev": "pnpm run --parallel dev",
|
||||||
|
"build": "pnpm run -r build",
|
||||||
|
"scraper": "pnpm --filter @poe2data/scraper dev",
|
||||||
|
"api": "pnpm --filter @poe2data/api dev",
|
||||||
|
"dashboard": "pnpm --filter @poe2data/dashboard dev",
|
||||||
|
"db:migrate": "pnpm --filter @poe2data/database migrate",
|
||||||
|
"db:studio": "pnpm --filter @poe2data/database studio"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"typescript": "^5.3.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
22
packages/api/package.json
Normal file
22
packages/api/package.json
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
{
|
||||||
|
"name": "@poe2data/api",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"build": "tsc",
|
||||||
|
"dev": "tsx watch src/index.ts",
|
||||||
|
"start": "node dist/index.js"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@poe2data/shared": "workspace:*",
|
||||||
|
"@poe2data/database": "workspace:*",
|
||||||
|
"fastify": "^5.0.0",
|
||||||
|
"@fastify/cors": "^10.0.0",
|
||||||
|
"drizzle-orm": "^0.38.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^20.0.0",
|
||||||
|
"tsx": "^4.7.0",
|
||||||
|
"typescript": "^5.3.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
61
packages/api/src/index.ts
Normal file
61
packages/api/src/index.ts
Normal file
|
|
@ -0,0 +1,61 @@
|
||||||
|
import Fastify from 'fastify';
|
||||||
|
import cors from '@fastify/cors';
|
||||||
|
import { itemRoutes } from './routes/items.js';
|
||||||
|
import { getDatabase, closeDatabase } from '@poe2data/database';
|
||||||
|
|
||||||
|
const PORT = parseInt(process.env.API_PORT || '3001', 10);
|
||||||
|
const HOST = process.env.API_HOST || '0.0.0.0';
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const fastify = Fastify({
|
||||||
|
logger: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Register CORS
|
||||||
|
await fastify.register(cors, {
|
||||||
|
origin: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Initialize database
|
||||||
|
getDatabase();
|
||||||
|
|
||||||
|
// Register routes
|
||||||
|
await fastify.register(itemRoutes, { prefix: '/api/v1' });
|
||||||
|
|
||||||
|
// Health check
|
||||||
|
fastify.get('/health', async () => ({ status: 'ok' }));
|
||||||
|
|
||||||
|
// Root endpoint
|
||||||
|
fastify.get('/', async () => ({
|
||||||
|
name: 'POE2 Data API',
|
||||||
|
version: '1.0.0',
|
||||||
|
endpoints: [
|
||||||
|
'GET /api/v1/items',
|
||||||
|
'GET /api/v1/items/:id/history',
|
||||||
|
'GET /api/v1/trends/movers',
|
||||||
|
'GET /api/v1/categories',
|
||||||
|
'GET /api/v1/league-start/priorities',
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Graceful shutdown
|
||||||
|
const shutdown = async () => {
|
||||||
|
console.log('Shutting down...');
|
||||||
|
await closeDatabase();
|
||||||
|
await fastify.close();
|
||||||
|
process.exit(0);
|
||||||
|
};
|
||||||
|
|
||||||
|
process.on('SIGINT', shutdown);
|
||||||
|
process.on('SIGTERM', shutdown);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await fastify.listen({ port: PORT, host: HOST });
|
||||||
|
console.log(`API server running at http://localhost:${PORT}`);
|
||||||
|
} catch (err) {
|
||||||
|
fastify.log.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main();
|
||||||
46
packages/api/src/routes/items.ts
Normal file
46
packages/api/src/routes/items.ts
Normal file
|
|
@ -0,0 +1,46 @@
|
||||||
|
import { FastifyInstance } from 'fastify';
|
||||||
|
import { TrendAnalysisService } from '../services/trend-analysis.js';
|
||||||
|
|
||||||
|
export async function itemRoutes(fastify: FastifyInstance) {
|
||||||
|
const trendService = new TrendAnalysisService();
|
||||||
|
|
||||||
|
// Get all items (optionally filtered by category)
|
||||||
|
fastify.get('/items', async (request, reply) => {
|
||||||
|
const { category } = request.query as { category?: string };
|
||||||
|
const items = await trendService.getAllItems(category);
|
||||||
|
return { items, count: items.length };
|
||||||
|
});
|
||||||
|
|
||||||
|
// Get item price history
|
||||||
|
fastify.get('/items/:id/history', async (request, reply) => {
|
||||||
|
const { id } = request.params as { id: string };
|
||||||
|
const history = await trendService.getItemHistory(parseInt(id, 10));
|
||||||
|
return { history };
|
||||||
|
});
|
||||||
|
|
||||||
|
// Get top movers
|
||||||
|
fastify.get('/trends/movers', async (request, reply) => {
|
||||||
|
const { limit } = request.query as { limit?: string };
|
||||||
|
const movers = await trendService.getTopMovers(parseInt(limit || '20', 10));
|
||||||
|
return movers;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Get category summaries
|
||||||
|
fastify.get('/categories', async (request, reply) => {
|
||||||
|
const summaries = await trendService.getCategorySummaries();
|
||||||
|
return { categories: summaries };
|
||||||
|
});
|
||||||
|
|
||||||
|
// Get league start priorities
|
||||||
|
fastify.get('/league-start/priorities', async (request, reply) => {
|
||||||
|
const priorities = await trendService.getLeagueStartPriorities();
|
||||||
|
return {
|
||||||
|
priorities,
|
||||||
|
summary: {
|
||||||
|
high: priorities.filter(p => p.priority === 'high').length,
|
||||||
|
medium: priorities.filter(p => p.priority === 'medium').length,
|
||||||
|
low: priorities.filter(p => p.priority === 'low').length,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
222
packages/api/src/services/trend-analysis.ts
Normal file
222
packages/api/src/services/trend-analysis.ts
Normal file
|
|
@ -0,0 +1,222 @@
|
||||||
|
import { eq, desc } from 'drizzle-orm';
|
||||||
|
import { getDatabase, items, priceHistory } from '@poe2data/database';
|
||||||
|
|
||||||
|
export interface TrendMover {
|
||||||
|
id: number;
|
||||||
|
name: string;
|
||||||
|
category: string;
|
||||||
|
iconUrl: string | null;
|
||||||
|
currentValue: number;
|
||||||
|
change7d: number;
|
||||||
|
volume: number;
|
||||||
|
chaosValue: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CategorySummary {
|
||||||
|
category: string;
|
||||||
|
itemCount: number;
|
||||||
|
topItem: string;
|
||||||
|
topValue: number;
|
||||||
|
avgChange7d: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LeagueStartPriority {
|
||||||
|
name: string;
|
||||||
|
category: string;
|
||||||
|
currentValue: number;
|
||||||
|
change7d: number;
|
||||||
|
volume: number;
|
||||||
|
priority: 'high' | 'medium' | 'low';
|
||||||
|
reason: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class TrendAnalysisService {
|
||||||
|
private db = getDatabase();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get all items with current prices
|
||||||
|
*/
|
||||||
|
async getAllItems(category?: string): Promise<TrendMover[]> {
|
||||||
|
// Get all items
|
||||||
|
const allItems = await this.db.select().from(items).all();
|
||||||
|
|
||||||
|
// Get latest price for each item - using Drizzle query builder
|
||||||
|
// We'll get all prices and pick the latest per item in JS
|
||||||
|
const allPrices = await this.db
|
||||||
|
.select()
|
||||||
|
.from(priceHistory)
|
||||||
|
.orderBy(desc(priceHistory.recordedAt))
|
||||||
|
.all();
|
||||||
|
|
||||||
|
// Group by item and get the latest
|
||||||
|
const priceMap = new Map<number, typeof allPrices[0]>();
|
||||||
|
for (const price of allPrices) {
|
||||||
|
if (!priceMap.has(price.itemId)) {
|
||||||
|
priceMap.set(price.itemId, price);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let result = allItems.map(item => {
|
||||||
|
const price = priceMap.get(item.id);
|
||||||
|
return {
|
||||||
|
id: item.id,
|
||||||
|
name: item.name,
|
||||||
|
category: item.category,
|
||||||
|
iconUrl: item.iconUrl,
|
||||||
|
currentValue: price?.divineValue || 0,
|
||||||
|
change7d: price?.change7d || 0,
|
||||||
|
volume: price?.volume || 0,
|
||||||
|
chaosValue: (price?.divineValue || 0) * (price?.chaosRate || 42),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
if (category) {
|
||||||
|
result = result.filter(i => i.category === category);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result.sort((a, b) => b.currentValue - a.currentValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get top movers (gainers and losers)
|
||||||
|
*/
|
||||||
|
async getTopMovers(limit: number = 20): Promise<{ gainers: TrendMover[]; losers: TrendMover[] }> {
|
||||||
|
const allItems = await this.getAllItems();
|
||||||
|
|
||||||
|
// Filter items with valid change data
|
||||||
|
const movers = allItems.filter(m => m.change7d !== 0 && m.currentValue > 0);
|
||||||
|
|
||||||
|
// Sort for gainers (highest positive change)
|
||||||
|
const gainers = [...movers]
|
||||||
|
.filter(m => m.change7d > 0)
|
||||||
|
.sort((a, b) => b.change7d - a.change7d)
|
||||||
|
.slice(0, limit);
|
||||||
|
|
||||||
|
// Sort for losers (most negative change)
|
||||||
|
const losers = [...movers]
|
||||||
|
.filter(m => m.change7d < 0)
|
||||||
|
.sort((a, b) => a.change7d - b.change7d)
|
||||||
|
.slice(0, limit);
|
||||||
|
|
||||||
|
return { gainers, losers };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get category summaries
|
||||||
|
*/
|
||||||
|
async getCategorySummaries(): Promise<CategorySummary[]> {
|
||||||
|
const allItems = await this.getAllItems();
|
||||||
|
|
||||||
|
const byCategory = new Map<string, TrendMover[]>();
|
||||||
|
for (const item of allItems) {
|
||||||
|
const list = byCategory.get(item.category) || [];
|
||||||
|
list.push(item);
|
||||||
|
byCategory.set(item.category, list);
|
||||||
|
}
|
||||||
|
|
||||||
|
const summaries: CategorySummary[] = [];
|
||||||
|
for (const [category, categoryItems] of byCategory) {
|
||||||
|
const sorted = categoryItems.sort((a, b) => b.currentValue - a.currentValue);
|
||||||
|
const validChanges = categoryItems.filter(i => i.change7d !== 0);
|
||||||
|
const avgChange = validChanges.length > 0
|
||||||
|
? validChanges.reduce((sum, i) => sum + i.change7d, 0) / validChanges.length
|
||||||
|
: 0;
|
||||||
|
|
||||||
|
summaries.push({
|
||||||
|
category,
|
||||||
|
itemCount: categoryItems.length,
|
||||||
|
topItem: sorted[0]?.name || 'N/A',
|
||||||
|
topValue: sorted[0]?.currentValue || 0,
|
||||||
|
avgChange7d: avgChange,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return summaries.sort((a, b) => b.topValue - a.topValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get league start priorities - items to farm early
|
||||||
|
*/
|
||||||
|
async getLeagueStartPriorities(): Promise<LeagueStartPriority[]> {
|
||||||
|
const allItems = await this.getAllItems();
|
||||||
|
|
||||||
|
const priorities: LeagueStartPriority[] = [];
|
||||||
|
|
||||||
|
for (const item of allItems) {
|
||||||
|
let priority: 'high' | 'medium' | 'low' = 'low';
|
||||||
|
let reason = '';
|
||||||
|
|
||||||
|
// High priority: High value + rising + good volume
|
||||||
|
if (item.currentValue > 1 && item.change7d > 20 && item.volume > 100) {
|
||||||
|
priority = 'high';
|
||||||
|
reason = 'High value, rising price, good demand';
|
||||||
|
}
|
||||||
|
// High priority: Very high value items
|
||||||
|
else if (item.currentValue > 50) {
|
||||||
|
priority = 'high';
|
||||||
|
reason = 'Very high value item';
|
||||||
|
}
|
||||||
|
// Medium priority: Rising prices
|
||||||
|
else if (item.change7d > 50) {
|
||||||
|
priority = 'medium';
|
||||||
|
reason = 'Rapidly rising price';
|
||||||
|
}
|
||||||
|
// Medium priority: Decent value with volume
|
||||||
|
else if (item.currentValue > 0.5 && item.volume > 500) {
|
||||||
|
priority = 'medium';
|
||||||
|
reason = 'Good value with high liquidity';
|
||||||
|
}
|
||||||
|
// Skip low value items
|
||||||
|
else if (item.currentValue < 0.01) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// Low priority: everything else valuable
|
||||||
|
else if (item.currentValue > 0.1) {
|
||||||
|
priority = 'low';
|
||||||
|
reason = 'Moderate value';
|
||||||
|
} else {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
priorities.push({
|
||||||
|
name: item.name,
|
||||||
|
category: item.category,
|
||||||
|
currentValue: item.currentValue,
|
||||||
|
change7d: item.change7d,
|
||||||
|
volume: item.volume,
|
||||||
|
priority,
|
||||||
|
reason,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by priority then value
|
||||||
|
const priorityOrder = { high: 0, medium: 1, low: 2 };
|
||||||
|
return priorities.sort((a, b) => {
|
||||||
|
const pDiff = priorityOrder[a.priority] - priorityOrder[b.priority];
|
||||||
|
if (pDiff !== 0) return pDiff;
|
||||||
|
return b.currentValue - a.currentValue;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get price history for an item
|
||||||
|
*/
|
||||||
|
async getItemHistory(itemId: number): Promise<{ timestamp: number; value: number; volume: number }[]> {
|
||||||
|
const history = await this.db
|
||||||
|
.select({
|
||||||
|
recordedAt: priceHistory.recordedAt,
|
||||||
|
divineValue: priceHistory.divineValue,
|
||||||
|
volume: priceHistory.volume,
|
||||||
|
})
|
||||||
|
.from(priceHistory)
|
||||||
|
.where(eq(priceHistory.itemId, itemId))
|
||||||
|
.orderBy(priceHistory.recordedAt)
|
||||||
|
.all();
|
||||||
|
|
||||||
|
return history.map(h => ({
|
||||||
|
timestamp: h.recordedAt?.getTime() || 0,
|
||||||
|
value: h.divineValue || 0,
|
||||||
|
volume: h.volume || 0,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
8
packages/api/tsconfig.json
Normal file
8
packages/api/tsconfig.json
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
{
|
||||||
|
"extends": "../../tsconfig.base.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"outDir": "dist",
|
||||||
|
"rootDir": "src"
|
||||||
|
},
|
||||||
|
"include": ["src/**/*"]
|
||||||
|
}
|
||||||
10
packages/database/drizzle.config.ts
Normal file
10
packages/database/drizzle.config.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
import { defineConfig } from 'drizzle-kit';
|
||||||
|
|
||||||
|
export default defineConfig({
|
||||||
|
schema: './src/schema/index.ts',
|
||||||
|
out: './drizzle',
|
||||||
|
dialect: 'sqlite',
|
||||||
|
dbCredentials: {
|
||||||
|
url: `file:${process.env.DATABASE_PATH || '../../data/poe2data.db'}`,
|
||||||
|
},
|
||||||
|
});
|
||||||
66
packages/database/drizzle/0000_perpetual_riptide.sql
Normal file
66
packages/database/drizzle/0000_perpetual_riptide.sql
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
CREATE TABLE `daily_prices` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`item_id` integer NOT NULL,
|
||||||
|
`date` text NOT NULL,
|
||||||
|
`open_value` real,
|
||||||
|
`close_value` real,
|
||||||
|
`high_value` real,
|
||||||
|
`low_value` real,
|
||||||
|
`avg_volume` real,
|
||||||
|
`chaos_rate` real,
|
||||||
|
FOREIGN KEY (`item_id`) REFERENCES `items`(`id`) ON UPDATE no action ON DELETE no action
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `idx_daily_prices_item_date` ON `daily_prices` (`item_id`,`date`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `items` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`external_id` text NOT NULL,
|
||||||
|
`details_id` text NOT NULL,
|
||||||
|
`league_id` integer NOT NULL,
|
||||||
|
`category` text NOT NULL,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`icon_url` text,
|
||||||
|
`created_at` integer,
|
||||||
|
FOREIGN KEY (`league_id`) REFERENCES `leagues`(`id`) ON UPDATE no action ON DELETE no action
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `idx_items_league_category` ON `items` (`league_id`,`category`);--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `idx_items_external_league` ON `items` (`external_id`,`league_id`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `leagues` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`display_name` text NOT NULL,
|
||||||
|
`start_date` integer,
|
||||||
|
`end_date` integer,
|
||||||
|
`is_active` integer DEFAULT true,
|
||||||
|
`created_at` integer
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `leagues_name_unique` ON `leagues` (`name`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `price_history` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`item_id` integer NOT NULL,
|
||||||
|
`snapshot_id` integer,
|
||||||
|
`divine_value` real,
|
||||||
|
`volume` real,
|
||||||
|
`change_7d` real,
|
||||||
|
`sparkline_data` text,
|
||||||
|
`exalted_rate` real,
|
||||||
|
`chaos_rate` real,
|
||||||
|
`recorded_at` integer NOT NULL,
|
||||||
|
FOREIGN KEY (`item_id`) REFERENCES `items`(`id`) ON UPDATE no action ON DELETE no action,
|
||||||
|
FOREIGN KEY (`snapshot_id`) REFERENCES `snapshots`(`id`) ON UPDATE no action ON DELETE no action
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `idx_price_history_item_recorded` ON `price_history` (`item_id`,`recorded_at`);--> statement-breakpoint
|
||||||
|
CREATE INDEX `idx_price_history_snapshot` ON `price_history` (`snapshot_id`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `snapshots` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`league_id` integer NOT NULL,
|
||||||
|
`category` text NOT NULL,
|
||||||
|
`scraped_at` integer NOT NULL,
|
||||||
|
`status` text DEFAULT 'pending',
|
||||||
|
`item_count` integer DEFAULT 0,
|
||||||
|
`error_message` text,
|
||||||
|
FOREIGN KEY (`league_id`) REFERENCES `leagues`(`id`) ON UPDATE no action ON DELETE no action
|
||||||
|
);
|
||||||
477
packages/database/drizzle/meta/0000_snapshot.json
Normal file
477
packages/database/drizzle/meta/0000_snapshot.json
Normal file
|
|
@ -0,0 +1,477 @@
|
||||||
|
{
|
||||||
|
"version": "6",
|
||||||
|
"dialect": "sqlite",
|
||||||
|
"id": "dc4861e0-d4dd-4c49-bcb1-fd4d5c4da8d4",
|
||||||
|
"prevId": "00000000-0000-0000-0000-000000000000",
|
||||||
|
"tables": {
|
||||||
|
"daily_prices": {
|
||||||
|
"name": "daily_prices",
|
||||||
|
"columns": {
|
||||||
|
"id": {
|
||||||
|
"name": "id",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": true,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": true
|
||||||
|
},
|
||||||
|
"item_id": {
|
||||||
|
"name": "item_id",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"date": {
|
||||||
|
"name": "date",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"open_value": {
|
||||||
|
"name": "open_value",
|
||||||
|
"type": "real",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"close_value": {
|
||||||
|
"name": "close_value",
|
||||||
|
"type": "real",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"high_value": {
|
||||||
|
"name": "high_value",
|
||||||
|
"type": "real",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"low_value": {
|
||||||
|
"name": "low_value",
|
||||||
|
"type": "real",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"avg_volume": {
|
||||||
|
"name": "avg_volume",
|
||||||
|
"type": "real",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"chaos_rate": {
|
||||||
|
"name": "chaos_rate",
|
||||||
|
"type": "real",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"indexes": {
|
||||||
|
"idx_daily_prices_item_date": {
|
||||||
|
"name": "idx_daily_prices_item_date",
|
||||||
|
"columns": [
|
||||||
|
"item_id",
|
||||||
|
"date"
|
||||||
|
],
|
||||||
|
"isUnique": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"foreignKeys": {
|
||||||
|
"daily_prices_item_id_items_id_fk": {
|
||||||
|
"name": "daily_prices_item_id_items_id_fk",
|
||||||
|
"tableFrom": "daily_prices",
|
||||||
|
"tableTo": "items",
|
||||||
|
"columnsFrom": [
|
||||||
|
"item_id"
|
||||||
|
],
|
||||||
|
"columnsTo": [
|
||||||
|
"id"
|
||||||
|
],
|
||||||
|
"onDelete": "no action",
|
||||||
|
"onUpdate": "no action"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"compositePrimaryKeys": {},
|
||||||
|
"uniqueConstraints": {},
|
||||||
|
"checkConstraints": {}
|
||||||
|
},
|
||||||
|
"items": {
|
||||||
|
"name": "items",
|
||||||
|
"columns": {
|
||||||
|
"id": {
|
||||||
|
"name": "id",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": true,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": true
|
||||||
|
},
|
||||||
|
"external_id": {
|
||||||
|
"name": "external_id",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"details_id": {
|
||||||
|
"name": "details_id",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"league_id": {
|
||||||
|
"name": "league_id",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"category": {
|
||||||
|
"name": "category",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"name": {
|
||||||
|
"name": "name",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"icon_url": {
|
||||||
|
"name": "icon_url",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"created_at": {
|
||||||
|
"name": "created_at",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"indexes": {
|
||||||
|
"idx_items_league_category": {
|
||||||
|
"name": "idx_items_league_category",
|
||||||
|
"columns": [
|
||||||
|
"league_id",
|
||||||
|
"category"
|
||||||
|
],
|
||||||
|
"isUnique": false
|
||||||
|
},
|
||||||
|
"idx_items_external_league": {
|
||||||
|
"name": "idx_items_external_league",
|
||||||
|
"columns": [
|
||||||
|
"external_id",
|
||||||
|
"league_id"
|
||||||
|
],
|
||||||
|
"isUnique": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"foreignKeys": {
|
||||||
|
"items_league_id_leagues_id_fk": {
|
||||||
|
"name": "items_league_id_leagues_id_fk",
|
||||||
|
"tableFrom": "items",
|
||||||
|
"tableTo": "leagues",
|
||||||
|
"columnsFrom": [
|
||||||
|
"league_id"
|
||||||
|
],
|
||||||
|
"columnsTo": [
|
||||||
|
"id"
|
||||||
|
],
|
||||||
|
"onDelete": "no action",
|
||||||
|
"onUpdate": "no action"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"compositePrimaryKeys": {},
|
||||||
|
"uniqueConstraints": {},
|
||||||
|
"checkConstraints": {}
|
||||||
|
},
|
||||||
|
"leagues": {
|
||||||
|
"name": "leagues",
|
||||||
|
"columns": {
|
||||||
|
"id": {
|
||||||
|
"name": "id",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": true,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": true
|
||||||
|
},
|
||||||
|
"name": {
|
||||||
|
"name": "name",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"display_name": {
|
||||||
|
"name": "display_name",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"start_date": {
|
||||||
|
"name": "start_date",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"end_date": {
|
||||||
|
"name": "end_date",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"is_active": {
|
||||||
|
"name": "is_active",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false,
|
||||||
|
"default": true
|
||||||
|
},
|
||||||
|
"created_at": {
|
||||||
|
"name": "created_at",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"indexes": {
|
||||||
|
"leagues_name_unique": {
|
||||||
|
"name": "leagues_name_unique",
|
||||||
|
"columns": [
|
||||||
|
"name"
|
||||||
|
],
|
||||||
|
"isUnique": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"foreignKeys": {},
|
||||||
|
"compositePrimaryKeys": {},
|
||||||
|
"uniqueConstraints": {},
|
||||||
|
"checkConstraints": {}
|
||||||
|
},
|
||||||
|
"price_history": {
|
||||||
|
"name": "price_history",
|
||||||
|
"columns": {
|
||||||
|
"id": {
|
||||||
|
"name": "id",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": true,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": true
|
||||||
|
},
|
||||||
|
"item_id": {
|
||||||
|
"name": "item_id",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"snapshot_id": {
|
||||||
|
"name": "snapshot_id",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"divine_value": {
|
||||||
|
"name": "divine_value",
|
||||||
|
"type": "real",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"volume": {
|
||||||
|
"name": "volume",
|
||||||
|
"type": "real",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"change_7d": {
|
||||||
|
"name": "change_7d",
|
||||||
|
"type": "real",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"sparkline_data": {
|
||||||
|
"name": "sparkline_data",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"exalted_rate": {
|
||||||
|
"name": "exalted_rate",
|
||||||
|
"type": "real",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"chaos_rate": {
|
||||||
|
"name": "chaos_rate",
|
||||||
|
"type": "real",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"recorded_at": {
|
||||||
|
"name": "recorded_at",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"indexes": {
|
||||||
|
"idx_price_history_item_recorded": {
|
||||||
|
"name": "idx_price_history_item_recorded",
|
||||||
|
"columns": [
|
||||||
|
"item_id",
|
||||||
|
"recorded_at"
|
||||||
|
],
|
||||||
|
"isUnique": false
|
||||||
|
},
|
||||||
|
"idx_price_history_snapshot": {
|
||||||
|
"name": "idx_price_history_snapshot",
|
||||||
|
"columns": [
|
||||||
|
"snapshot_id"
|
||||||
|
],
|
||||||
|
"isUnique": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"foreignKeys": {
|
||||||
|
"price_history_item_id_items_id_fk": {
|
||||||
|
"name": "price_history_item_id_items_id_fk",
|
||||||
|
"tableFrom": "price_history",
|
||||||
|
"tableTo": "items",
|
||||||
|
"columnsFrom": [
|
||||||
|
"item_id"
|
||||||
|
],
|
||||||
|
"columnsTo": [
|
||||||
|
"id"
|
||||||
|
],
|
||||||
|
"onDelete": "no action",
|
||||||
|
"onUpdate": "no action"
|
||||||
|
},
|
||||||
|
"price_history_snapshot_id_snapshots_id_fk": {
|
||||||
|
"name": "price_history_snapshot_id_snapshots_id_fk",
|
||||||
|
"tableFrom": "price_history",
|
||||||
|
"tableTo": "snapshots",
|
||||||
|
"columnsFrom": [
|
||||||
|
"snapshot_id"
|
||||||
|
],
|
||||||
|
"columnsTo": [
|
||||||
|
"id"
|
||||||
|
],
|
||||||
|
"onDelete": "no action",
|
||||||
|
"onUpdate": "no action"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"compositePrimaryKeys": {},
|
||||||
|
"uniqueConstraints": {},
|
||||||
|
"checkConstraints": {}
|
||||||
|
},
|
||||||
|
"snapshots": {
|
||||||
|
"name": "snapshots",
|
||||||
|
"columns": {
|
||||||
|
"id": {
|
||||||
|
"name": "id",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": true,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": true
|
||||||
|
},
|
||||||
|
"league_id": {
|
||||||
|
"name": "league_id",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"category": {
|
||||||
|
"name": "category",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"scraped_at": {
|
||||||
|
"name": "scraped_at",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"name": "status",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false,
|
||||||
|
"default": "'pending'"
|
||||||
|
},
|
||||||
|
"item_count": {
|
||||||
|
"name": "item_count",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false,
|
||||||
|
"default": 0
|
||||||
|
},
|
||||||
|
"error_message": {
|
||||||
|
"name": "error_message",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"indexes": {},
|
||||||
|
"foreignKeys": {
|
||||||
|
"snapshots_league_id_leagues_id_fk": {
|
||||||
|
"name": "snapshots_league_id_leagues_id_fk",
|
||||||
|
"tableFrom": "snapshots",
|
||||||
|
"tableTo": "leagues",
|
||||||
|
"columnsFrom": [
|
||||||
|
"league_id"
|
||||||
|
],
|
||||||
|
"columnsTo": [
|
||||||
|
"id"
|
||||||
|
],
|
||||||
|
"onDelete": "no action",
|
||||||
|
"onUpdate": "no action"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"compositePrimaryKeys": {},
|
||||||
|
"uniqueConstraints": {},
|
||||||
|
"checkConstraints": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"views": {},
|
||||||
|
"enums": {},
|
||||||
|
"_meta": {
|
||||||
|
"schemas": {},
|
||||||
|
"tables": {},
|
||||||
|
"columns": {}
|
||||||
|
},
|
||||||
|
"internal": {
|
||||||
|
"indexes": {}
|
||||||
|
}
|
||||||
|
}
|
||||||
13
packages/database/drizzle/meta/_journal.json
Normal file
13
packages/database/drizzle/meta/_journal.json
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
{
|
||||||
|
"version": "7",
|
||||||
|
"dialect": "sqlite",
|
||||||
|
"entries": [
|
||||||
|
{
|
||||||
|
"idx": 0,
|
||||||
|
"version": "6",
|
||||||
|
"when": 1770307380064,
|
||||||
|
"tag": "0000_perpetual_riptide",
|
||||||
|
"breakpoints": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
32
packages/database/package.json
Normal file
32
packages/database/package.json
Normal file
|
|
@ -0,0 +1,32 @@
|
||||||
|
{
|
||||||
|
"name": "@poe2data/database",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"main": "./dist/index.js",
|
||||||
|
"types": "./dist/index.d.ts",
|
||||||
|
"exports": {
|
||||||
|
".": {
|
||||||
|
"import": "./dist/index.js",
|
||||||
|
"types": "./dist/index.d.ts"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"scripts": {
|
||||||
|
"build": "tsc",
|
||||||
|
"dev": "tsc --watch",
|
||||||
|
"migrate": "tsx src/migrate.ts",
|
||||||
|
"migrate:drizzle": "drizzle-kit migrate",
|
||||||
|
"generate": "drizzle-kit generate",
|
||||||
|
"studio": "drizzle-kit studio"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@poe2data/shared": "workspace:*",
|
||||||
|
"@libsql/client": "^0.14.0",
|
||||||
|
"drizzle-orm": "^0.38.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^20.0.0",
|
||||||
|
"drizzle-kit": "^0.30.0",
|
||||||
|
"tsx": "^4.7.0",
|
||||||
|
"typescript": "^5.3.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
39
packages/database/src/index.ts
Normal file
39
packages/database/src/index.ts
Normal file
|
|
@ -0,0 +1,39 @@
|
||||||
|
import { createClient } from '@libsql/client';
|
||||||
|
import { drizzle } from 'drizzle-orm/libsql';
|
||||||
|
import * as schema from './schema/index.js';
|
||||||
|
import path from 'path';
|
||||||
|
import { fileURLToPath } from 'url';
|
||||||
|
|
||||||
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||||
|
|
||||||
|
// Default database path
|
||||||
|
const DEFAULT_DB_PATH = path.resolve(__dirname, '../../..', 'data', 'poe2data.db');
|
||||||
|
|
||||||
|
let dbInstance: ReturnType<typeof drizzle> | null = null;
|
||||||
|
let clientInstance: ReturnType<typeof createClient> | null = null;
|
||||||
|
|
||||||
|
export function getDatabase(dbPath?: string) {
|
||||||
|
if (!dbInstance) {
|
||||||
|
const finalPath = dbPath || process.env.DATABASE_PATH || DEFAULT_DB_PATH;
|
||||||
|
console.log(`Opening database at: ${finalPath}`);
|
||||||
|
|
||||||
|
clientInstance = createClient({
|
||||||
|
url: `file:${finalPath}`,
|
||||||
|
});
|
||||||
|
|
||||||
|
dbInstance = drizzle(clientInstance, { schema });
|
||||||
|
}
|
||||||
|
return dbInstance;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function closeDatabase() {
|
||||||
|
if (clientInstance) {
|
||||||
|
clientInstance.close();
|
||||||
|
clientInstance = null;
|
||||||
|
dbInstance = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Re-export schema and types
|
||||||
|
export * from './schema/index.js';
|
||||||
|
export { schema };
|
||||||
42
packages/database/src/migrate.ts
Normal file
42
packages/database/src/migrate.ts
Normal file
|
|
@ -0,0 +1,42 @@
|
||||||
|
import { createClient } from '@libsql/client';
|
||||||
|
import { drizzle } from 'drizzle-orm/libsql';
|
||||||
|
import { migrate } from 'drizzle-orm/libsql/migrator';
|
||||||
|
import path from 'path';
|
||||||
|
import { fileURLToPath } from 'url';
|
||||||
|
import fs from 'fs';
|
||||||
|
|
||||||
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||||
|
|
||||||
|
const DB_PATH = process.env.DATABASE_PATH || path.resolve(__dirname, '../../..', 'data', 'poe2data.db');
|
||||||
|
const MIGRATIONS_PATH = path.resolve(__dirname, '..', 'drizzle');
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
console.log('Running database migrations...');
|
||||||
|
console.log(`Database path: ${DB_PATH}`);
|
||||||
|
console.log(`Migrations path: ${MIGRATIONS_PATH}`);
|
||||||
|
|
||||||
|
// Ensure data directory exists
|
||||||
|
const dataDir = path.dirname(DB_PATH);
|
||||||
|
if (!fs.existsSync(dataDir)) {
|
||||||
|
fs.mkdirSync(dataDir, { recursive: true });
|
||||||
|
console.log(`Created data directory: ${dataDir}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = createClient({
|
||||||
|
url: `file:${DB_PATH}`,
|
||||||
|
});
|
||||||
|
|
||||||
|
const db = drizzle(client);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await migrate(db, { migrationsFolder: MIGRATIONS_PATH });
|
||||||
|
console.log('Migrations completed successfully!');
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Migration failed:', error);
|
||||||
|
process.exit(1);
|
||||||
|
} finally {
|
||||||
|
client.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main();
|
||||||
95
packages/database/src/schema/index.ts
Normal file
95
packages/database/src/schema/index.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
||||||
|
import { sqliteTable, text, integer, real, index, uniqueIndex } from 'drizzle-orm/sqlite-core';
|
||||||
|
|
||||||
|
// Leagues table
|
||||||
|
export const leagues = sqliteTable('leagues', {
|
||||||
|
id: integer('id').primaryKey({ autoIncrement: true }),
|
||||||
|
name: text('name').notNull().unique(),
|
||||||
|
displayName: text('display_name').notNull(),
|
||||||
|
startDate: integer('start_date', { mode: 'timestamp' }),
|
||||||
|
endDate: integer('end_date', { mode: 'timestamp' }),
|
||||||
|
isActive: integer('is_active', { mode: 'boolean' }).default(true),
|
||||||
|
createdAt: integer('created_at', { mode: 'timestamp' }).$defaultFn(() => new Date()),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Items table - stores item metadata
|
||||||
|
export const items = sqliteTable('items', {
|
||||||
|
id: integer('id').primaryKey({ autoIncrement: true }),
|
||||||
|
externalId: text('external_id').notNull(), // poe.ninja ID (e.g., "divine", "chaos")
|
||||||
|
detailsId: text('details_id').notNull(), // poe.ninja details ID (e.g., "divine-orb")
|
||||||
|
leagueId: integer('league_id').references(() => leagues.id).notNull(),
|
||||||
|
category: text('category').notNull(), // Currency, Fragments, etc.
|
||||||
|
name: text('name').notNull(),
|
||||||
|
iconUrl: text('icon_url'),
|
||||||
|
createdAt: integer('created_at', { mode: 'timestamp' }).$defaultFn(() => new Date()),
|
||||||
|
}, (table) => [
|
||||||
|
index('idx_items_league_category').on(table.leagueId, table.category),
|
||||||
|
uniqueIndex('idx_items_external_league').on(table.externalId, table.leagueId),
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Snapshots table - tracks when we scraped data
|
||||||
|
export const snapshots = sqliteTable('snapshots', {
|
||||||
|
id: integer('id').primaryKey({ autoIncrement: true }),
|
||||||
|
leagueId: integer('league_id').references(() => leagues.id).notNull(),
|
||||||
|
category: text('category').notNull(),
|
||||||
|
scrapedAt: integer('scraped_at', { mode: 'timestamp' }).notNull().$defaultFn(() => new Date()),
|
||||||
|
status: text('status', { enum: ['pending', 'success', 'failed'] }).default('pending'),
|
||||||
|
itemCount: integer('item_count').default(0),
|
||||||
|
errorMessage: text('error_message'),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Price history table - stores historical prices
|
||||||
|
export const priceHistory = sqliteTable('price_history', {
|
||||||
|
id: integer('id').primaryKey({ autoIncrement: true }),
|
||||||
|
itemId: integer('item_id').references(() => items.id).notNull(),
|
||||||
|
snapshotId: integer('snapshot_id').references(() => snapshots.id),
|
||||||
|
|
||||||
|
// Value in divines (primary currency)
|
||||||
|
divineValue: real('divine_value'),
|
||||||
|
|
||||||
|
// Trading volume
|
||||||
|
volume: real('volume'),
|
||||||
|
|
||||||
|
// 7-day change percentage
|
||||||
|
change7d: real('change_7d'),
|
||||||
|
|
||||||
|
// Sparkline data (JSON array of 7 values)
|
||||||
|
sparklineData: text('sparkline_data'),
|
||||||
|
|
||||||
|
// Exchange rates at time of snapshot
|
||||||
|
exaltedRate: real('exalted_rate'), // exalted per divine
|
||||||
|
chaosRate: real('chaos_rate'), // chaos per divine
|
||||||
|
|
||||||
|
recordedAt: integer('recorded_at', { mode: 'timestamp' }).notNull().$defaultFn(() => new Date()),
|
||||||
|
}, (table) => [
|
||||||
|
index('idx_price_history_item_recorded').on(table.itemId, table.recordedAt),
|
||||||
|
index('idx_price_history_snapshot').on(table.snapshotId),
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Daily price aggregates - for faster trend queries
|
||||||
|
export const dailyPrices = sqliteTable('daily_prices', {
|
||||||
|
id: integer('id').primaryKey({ autoIncrement: true }),
|
||||||
|
itemId: integer('item_id').references(() => items.id).notNull(),
|
||||||
|
date: text('date').notNull(), // YYYY-MM-DD format
|
||||||
|
|
||||||
|
openValue: real('open_value'),
|
||||||
|
closeValue: real('close_value'),
|
||||||
|
highValue: real('high_value'),
|
||||||
|
lowValue: real('low_value'),
|
||||||
|
avgVolume: real('avg_volume'),
|
||||||
|
|
||||||
|
chaosRate: real('chaos_rate'), // chaos per divine for that day
|
||||||
|
}, (table) => [
|
||||||
|
uniqueIndex('idx_daily_prices_item_date').on(table.itemId, table.date),
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Type exports for use in other packages
|
||||||
|
export type League = typeof leagues.$inferSelect;
|
||||||
|
export type NewLeague = typeof leagues.$inferInsert;
|
||||||
|
export type Item = typeof items.$inferSelect;
|
||||||
|
export type NewItem = typeof items.$inferInsert;
|
||||||
|
export type Snapshot = typeof snapshots.$inferSelect;
|
||||||
|
export type NewSnapshot = typeof snapshots.$inferInsert;
|
||||||
|
export type PriceHistory = typeof priceHistory.$inferSelect;
|
||||||
|
export type NewPriceHistory = typeof priceHistory.$inferInsert;
|
||||||
|
export type DailyPrice = typeof dailyPrices.$inferSelect;
|
||||||
|
export type NewDailyPrice = typeof dailyPrices.$inferInsert;
|
||||||
8
packages/database/tsconfig.json
Normal file
8
packages/database/tsconfig.json
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
{
|
||||||
|
"extends": "../../tsconfig.base.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"outDir": "dist",
|
||||||
|
"rootDir": "src"
|
||||||
|
},
|
||||||
|
"include": ["src/**/*"]
|
||||||
|
}
|
||||||
22
packages/scraper/package.json
Normal file
22
packages/scraper/package.json
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
{
|
||||||
|
"name": "@poe2data/scraper",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"main": "./dist/index.js",
|
||||||
|
"scripts": {
|
||||||
|
"build": "tsc",
|
||||||
|
"dev": "tsx src/index.ts",
|
||||||
|
"start": "node dist/index.js"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@poe2data/shared": "workspace:*",
|
||||||
|
"@poe2data/database": "workspace:*",
|
||||||
|
"drizzle-orm": "^0.38.0",
|
||||||
|
"node-cron": "^3.0.3"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^20.0.0",
|
||||||
|
"tsx": "^4.7.0",
|
||||||
|
"typescript": "^5.3.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
232
packages/scraper/src/client/poe-ninja.ts
Normal file
232
packages/scraper/src/client/poe-ninja.ts
Normal file
|
|
@ -0,0 +1,232 @@
|
||||||
|
import type {
|
||||||
|
PoeNinjaSearchResponse,
|
||||||
|
PoeNinjaOverviewResponse,
|
||||||
|
PoeNinjaDetailsResponse,
|
||||||
|
} from '@poe2data/shared';
|
||||||
|
import { CURRENT_LEAGUE } from '@poe2data/shared';
|
||||||
|
|
||||||
|
const BASE_URL = 'https://poe.ninja/poe2/api/economy/exchange/current';
|
||||||
|
|
||||||
|
// Cache for item details keyed by "category:id"
|
||||||
|
const itemDetailsCache = new Map<string, { name: string; icon: string }>();
|
||||||
|
|
||||||
|
// Simple rate limiter
|
||||||
|
class RateLimiter {
|
||||||
|
private tokens: number;
|
||||||
|
private lastRefill: number;
|
||||||
|
private readonly maxTokens: number;
|
||||||
|
private readonly refillRate: number;
|
||||||
|
|
||||||
|
constructor(maxTokens = 5, refillRateMs = 1000) {
|
||||||
|
this.maxTokens = maxTokens;
|
||||||
|
this.refillRate = refillRateMs;
|
||||||
|
this.tokens = maxTokens;
|
||||||
|
this.lastRefill = Date.now();
|
||||||
|
}
|
||||||
|
|
||||||
|
private refill(): void {
|
||||||
|
const now = Date.now();
|
||||||
|
const elapsed = now - this.lastRefill;
|
||||||
|
const tokensToAdd = Math.floor(elapsed / this.refillRate);
|
||||||
|
if (tokensToAdd > 0) {
|
||||||
|
this.tokens = Math.min(this.maxTokens, this.tokens + tokensToAdd);
|
||||||
|
this.lastRefill = now;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async acquire(): Promise<void> {
|
||||||
|
this.refill();
|
||||||
|
if (this.tokens <= 0) {
|
||||||
|
const waitTime = this.refillRate - (Date.now() - this.lastRefill);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, waitTime));
|
||||||
|
this.refill();
|
||||||
|
}
|
||||||
|
this.tokens--;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class PoeNinjaClient {
|
||||||
|
private rateLimiter: RateLimiter;
|
||||||
|
private league: string;
|
||||||
|
|
||||||
|
constructor(league: string = CURRENT_LEAGUE) {
|
||||||
|
this.rateLimiter = new RateLimiter(5, 500); // 5 requests per 500ms
|
||||||
|
this.league = league;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async fetch<T>(url: string): Promise<T> {
|
||||||
|
await this.rateLimiter.acquire();
|
||||||
|
|
||||||
|
const response = await fetch(url, {
|
||||||
|
headers: {
|
||||||
|
'User-Agent': 'POE2Data Scraper/1.0',
|
||||||
|
Accept: 'application/json',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.json() as Promise<T>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get all items across all categories
|
||||||
|
*/
|
||||||
|
async search(): Promise<PoeNinjaSearchResponse> {
|
||||||
|
const url = `${BASE_URL}/search?league=${encodeURIComponent(this.league)}`;
|
||||||
|
return this.fetch<PoeNinjaSearchResponse>(url);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get items for a specific category
|
||||||
|
*/
|
||||||
|
async getOverview(type: string): Promise<PoeNinjaOverviewResponse> {
|
||||||
|
const url = `${BASE_URL}/overview?league=${encodeURIComponent(this.league)}&type=${encodeURIComponent(type)}`;
|
||||||
|
return this.fetch<PoeNinjaOverviewResponse>(url);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get detailed info for a specific item
|
||||||
|
*/
|
||||||
|
async getDetails(type: string, id: string): Promise<PoeNinjaDetailsResponse> {
|
||||||
|
const url = `${BASE_URL}/details?league=${encodeURIComponent(this.league)}&type=${encodeURIComponent(type)}&id=${encodeURIComponent(id)}`;
|
||||||
|
return this.fetch<PoeNinjaDetailsResponse>(url);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set the league for subsequent requests
|
||||||
|
*/
|
||||||
|
setLeague(league: string): void {
|
||||||
|
this.league = league;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get item name and icon from cache
|
||||||
|
* Returns ID as fallback name if not in cache
|
||||||
|
*/
|
||||||
|
getItemInfo(
|
||||||
|
type: string,
|
||||||
|
id: string
|
||||||
|
): { name: string; icon: string } {
|
||||||
|
const cacheKey = `${type}:${id}`;
|
||||||
|
|
||||||
|
// Check cache first
|
||||||
|
const cached = itemDetailsCache.get(cacheKey);
|
||||||
|
if (cached) {
|
||||||
|
return cached;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Not in cache, use ID as fallback name
|
||||||
|
// Don't call API as the details endpoint uses different ID format
|
||||||
|
const fallback = { name: id, icon: '' };
|
||||||
|
itemDetailsCache.set(cacheKey, fallback);
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Preload item names from search endpoint
|
||||||
|
* This populates the cache with all known items, using multiple key variations
|
||||||
|
*/
|
||||||
|
async preloadItemNames(): Promise<void> {
|
||||||
|
console.log('Preloading item names from search endpoint...');
|
||||||
|
const searchData = await this.search();
|
||||||
|
|
||||||
|
let count = 0;
|
||||||
|
// The search response has items grouped by category type
|
||||||
|
for (const [category, searchItems] of Object.entries(searchData.items)) {
|
||||||
|
for (const item of searchItems) {
|
||||||
|
const itemInfo = {
|
||||||
|
name: item.name,
|
||||||
|
icon: item.icon || '',
|
||||||
|
};
|
||||||
|
|
||||||
|
// Generate multiple possible keys for this item
|
||||||
|
const keys = this.generatePossibleKeys(item.name);
|
||||||
|
for (const key of keys) {
|
||||||
|
const cacheKey = `${category}:${key}`;
|
||||||
|
if (!itemDetailsCache.has(cacheKey)) {
|
||||||
|
itemDetailsCache.set(cacheKey, itemInfo);
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.log(`Preloaded ${count} cache entries from ${Object.keys(searchData.items).length} categories`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate possible abbreviated keys for an item name
|
||||||
|
* e.g., "Orb of Alchemy" -> ["orb-of-alchemy", "alchemy", "alch"]
|
||||||
|
*/
|
||||||
|
private generatePossibleKeys(name: string): string[] {
|
||||||
|
const keys: string[] = [];
|
||||||
|
const lower = name.toLowerCase();
|
||||||
|
|
||||||
|
// Full slugified name
|
||||||
|
keys.push(this.slugify(name));
|
||||||
|
|
||||||
|
// Common POE abbreviation patterns
|
||||||
|
// "Orb of X" -> "x" (first word of X)
|
||||||
|
const orbOfMatch = lower.match(/^orb of (\w+)/);
|
||||||
|
if (orbOfMatch) {
|
||||||
|
keys.push(orbOfMatch[1]);
|
||||||
|
// Also add 4-5 letter prefixes as abbreviations
|
||||||
|
if (orbOfMatch[1].length > 4) {
|
||||||
|
keys.push(orbOfMatch[1].substring(0, 4));
|
||||||
|
keys.push(orbOfMatch[1].substring(0, 5));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// "X Orb" -> "x" (first word)
|
||||||
|
const xOrbMatch = lower.match(/^(\w+)(?:'s)? orb/);
|
||||||
|
if (xOrbMatch) {
|
||||||
|
keys.push(xOrbMatch[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// "X's Y" -> "x", "y", "xs"
|
||||||
|
const possessiveMatch = lower.match(/^(\w+)'s (\w+)/);
|
||||||
|
if (possessiveMatch) {
|
||||||
|
keys.push(possessiveMatch[1]);
|
||||||
|
keys.push(possessiveMatch[2]);
|
||||||
|
keys.push(possessiveMatch[1] + 's');
|
||||||
|
}
|
||||||
|
|
||||||
|
// First word alone
|
||||||
|
const firstWord = lower.split(/\s+/)[0].replace(/[^a-z]/g, '');
|
||||||
|
if (firstWord.length >= 3) {
|
||||||
|
keys.push(firstWord);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Special abbreviations for common items
|
||||||
|
const specialMappings: Record<string, string> = {
|
||||||
|
'gemcutter\'s prism': 'gcp',
|
||||||
|
'glassblower\'s bauble': 'bauble',
|
||||||
|
'armourer\'s scrap': 'scrap',
|
||||||
|
'blacksmith\'s whetstone': 'whetstone',
|
||||||
|
'scroll of wisdom': 'wisdom',
|
||||||
|
'orb of transmutation': 'transmute',
|
||||||
|
'orb of regret': 'regret',
|
||||||
|
'vaal orb': 'vaal',
|
||||||
|
'mirror of kalandra': 'mirror',
|
||||||
|
'regal orb': 'regal',
|
||||||
|
'arcanist\'s etcher': 'etcher',
|
||||||
|
};
|
||||||
|
if (specialMappings[lower]) {
|
||||||
|
keys.push(specialMappings[lower]);
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...new Set(keys)];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert item name to URL slug format
|
||||||
|
*/
|
||||||
|
private slugify(name: string): string {
|
||||||
|
return name
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^a-z0-9]+/g, '-')
|
||||||
|
.replace(/^-|-$/g, '');
|
||||||
|
}
|
||||||
|
}
|
||||||
86
packages/scraper/src/index.ts
Normal file
86
packages/scraper/src/index.ts
Normal file
|
|
@ -0,0 +1,86 @@
|
||||||
|
import { PoeNinjaClient } from './client/poe-ninja.js';
|
||||||
|
import { DataSaver } from './services/data-saver.js';
|
||||||
|
import { Scheduler } from './scheduler.js';
|
||||||
|
import { POE2_CATEGORIES, CURRENT_LEAGUE } from '@poe2data/shared';
|
||||||
|
import { closeDatabase } from '@poe2data/database';
|
||||||
|
import { itemNameMapper } from './services/item-name-mapper.js';
|
||||||
|
|
||||||
|
const SCHEDULED_MODE = process.argv.includes('--scheduled') || process.argv.includes('-s');
|
||||||
|
|
||||||
|
async function runOnce() {
|
||||||
|
console.log(`POE2 Data Scraper - League: ${CURRENT_LEAGUE}`);
|
||||||
|
console.log('='.repeat(50));
|
||||||
|
|
||||||
|
const client = new PoeNinjaClient();
|
||||||
|
const saver = new DataSaver();
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Initialize item name mapper from search endpoint
|
||||||
|
console.log('Initializing item name mapper...');
|
||||||
|
const searchData = await client.search();
|
||||||
|
itemNameMapper.initialize(searchData);
|
||||||
|
|
||||||
|
const leagueId = await saver.ensureLeague(CURRENT_LEAGUE);
|
||||||
|
console.log(`League ID: ${leagueId}`);
|
||||||
|
|
||||||
|
for (const category of POE2_CATEGORIES) {
|
||||||
|
console.log(`\nScraping ${category.title} (${category.type})...`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const data = await client.getOverview(category.type);
|
||||||
|
await saver.saveCurrencyOverview(data, category.title, category.type, leagueId);
|
||||||
|
|
||||||
|
const sorted = [...data.lines]
|
||||||
|
.sort((a, b) => (b.primaryValue || 0) - (a.primaryValue || 0))
|
||||||
|
.slice(0, 5);
|
||||||
|
|
||||||
|
const itemMap = new Map(data.core.items.map((i) => [i.id, i]));
|
||||||
|
|
||||||
|
console.log(` Top 5 by value:`);
|
||||||
|
for (const line of sorted) {
|
||||||
|
const item = itemMap.get(line.id);
|
||||||
|
const name = item?.name || line.id;
|
||||||
|
const change = line.sparkline?.totalChange ?? 0;
|
||||||
|
const changeStr = change >= 0 ? `+${change.toFixed(1)}%` : `${change.toFixed(1)}%`;
|
||||||
|
console.log(
|
||||||
|
` ${name.substring(0, 30).padEnd(32)} ${line.primaryValue.toFixed(4)} div [7d: ${changeStr}]`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error(` Failed to scrape ${category.title}:`, err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('\n' + '='.repeat(50));
|
||||||
|
console.log('Scraping completed successfully!');
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error:', error);
|
||||||
|
process.exit(1);
|
||||||
|
} finally {
|
||||||
|
await closeDatabase();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runScheduled() {
|
||||||
|
console.log(`POE2 Data Scraper - SCHEDULED MODE`);
|
||||||
|
console.log(`League: ${CURRENT_LEAGUE}`);
|
||||||
|
console.log('='.repeat(50));
|
||||||
|
|
||||||
|
const scheduler = new Scheduler();
|
||||||
|
|
||||||
|
// Handle shutdown
|
||||||
|
process.on('SIGINT', async () => {
|
||||||
|
console.log('\nShutting down scheduler...');
|
||||||
|
await closeDatabase();
|
||||||
|
process.exit(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
scheduler.start();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Main entry point
|
||||||
|
if (SCHEDULED_MODE) {
|
||||||
|
runScheduled();
|
||||||
|
} else {
|
||||||
|
runOnce();
|
||||||
|
}
|
||||||
76
packages/scraper/src/scheduler.ts
Normal file
76
packages/scraper/src/scheduler.ts
Normal file
|
|
@ -0,0 +1,76 @@
|
||||||
|
import cron from 'node-cron';
|
||||||
|
import { PoeNinjaClient } from './client/poe-ninja.js';
|
||||||
|
import { DataSaver } from './services/data-saver.js';
|
||||||
|
import { POE2_CATEGORIES, CURRENT_LEAGUE } from '@poe2data/shared';
|
||||||
|
import { itemNameMapper } from './services/item-name-mapper.js';
|
||||||
|
|
||||||
|
export class Scheduler {
|
||||||
|
private client: PoeNinjaClient;
|
||||||
|
private saver: DataSaver;
|
||||||
|
private leagueId: number | null = null;
|
||||||
|
private isRunning = false;
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
this.client = new PoeNinjaClient();
|
||||||
|
this.saver = new DataSaver();
|
||||||
|
}
|
||||||
|
|
||||||
|
async initialize() {
|
||||||
|
// Initialize item name mapper from search endpoint
|
||||||
|
console.log('Initializing item name mapper...');
|
||||||
|
const searchData = await this.client.search();
|
||||||
|
itemNameMapper.initialize(searchData);
|
||||||
|
|
||||||
|
this.leagueId = await this.saver.ensureLeague(CURRENT_LEAGUE);
|
||||||
|
console.log(`Scheduler initialized for league: ${CURRENT_LEAGUE} (ID: ${this.leagueId})`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async scrapeAll() {
|
||||||
|
if (this.isRunning) {
|
||||||
|
console.log('Scrape already in progress, skipping...');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!this.leagueId) {
|
||||||
|
await this.initialize();
|
||||||
|
}
|
||||||
|
|
||||||
|
this.isRunning = true;
|
||||||
|
const startTime = Date.now();
|
||||||
|
console.log(`\n[${ new Date().toISOString() }] Starting scrape...`);
|
||||||
|
|
||||||
|
let successCount = 0;
|
||||||
|
let errorCount = 0;
|
||||||
|
|
||||||
|
for (const category of POE2_CATEGORIES) {
|
||||||
|
try {
|
||||||
|
const data = await this.client.getOverview(category.type);
|
||||||
|
await this.saver.saveCurrencyOverview(data, category.title, category.type, this.leagueId!);
|
||||||
|
successCount++;
|
||||||
|
} catch (err) {
|
||||||
|
console.error(` Failed to scrape ${category.title}:`, err);
|
||||||
|
errorCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const duration = ((Date.now() - startTime) / 1000).toFixed(1);
|
||||||
|
console.log(`Scrape completed in ${duration}s: ${successCount} success, ${errorCount} errors\n`);
|
||||||
|
|
||||||
|
this.isRunning = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
start() {
|
||||||
|
console.log('Starting scheduler...');
|
||||||
|
console.log('Schedule: Every 30 minutes');
|
||||||
|
|
||||||
|
// Run immediately on start
|
||||||
|
this.scrapeAll();
|
||||||
|
|
||||||
|
// Schedule every 30 minutes
|
||||||
|
cron.schedule('*/30 * * * *', () => {
|
||||||
|
this.scrapeAll();
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('Scheduler started. Press Ctrl+C to stop.');
|
||||||
|
}
|
||||||
|
}
|
||||||
141
packages/scraper/src/services/data-saver.ts
Normal file
141
packages/scraper/src/services/data-saver.ts
Normal file
|
|
@ -0,0 +1,141 @@
|
||||||
|
import { eq, and } from 'drizzle-orm';
|
||||||
|
import {
|
||||||
|
getDatabase,
|
||||||
|
leagues,
|
||||||
|
items,
|
||||||
|
snapshots,
|
||||||
|
priceHistory,
|
||||||
|
} from '@poe2data/database';
|
||||||
|
import type { PoeNinjaOverviewResponse } from '@poe2data/shared';
|
||||||
|
import { CURRENT_LEAGUE } from '@poe2data/shared';
|
||||||
|
import { itemNameMapper, type ItemNameMapper } from './item-name-mapper.js';
|
||||||
|
|
||||||
|
export class DataSaver {
|
||||||
|
private db = getDatabase();
|
||||||
|
private mapper: ItemNameMapper = itemNameMapper;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure the league exists in the database
|
||||||
|
*/
|
||||||
|
async ensureLeague(leagueName: string = CURRENT_LEAGUE): Promise<number> {
|
||||||
|
const existing = await this.db
|
||||||
|
.select()
|
||||||
|
.from(leagues)
|
||||||
|
.where(eq(leagues.name, leagueName))
|
||||||
|
.get();
|
||||||
|
|
||||||
|
if (existing) {
|
||||||
|
return existing.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await this.db
|
||||||
|
.insert(leagues)
|
||||||
|
.values({
|
||||||
|
name: leagueName,
|
||||||
|
displayName: leagueName,
|
||||||
|
isActive: true,
|
||||||
|
})
|
||||||
|
.returning({ id: leagues.id });
|
||||||
|
|
||||||
|
return result[0].id;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Save currency overview data to the database
|
||||||
|
*/
|
||||||
|
async saveCurrencyOverview(
|
||||||
|
data: PoeNinjaOverviewResponse,
|
||||||
|
category: string,
|
||||||
|
categoryType: string,
|
||||||
|
leagueId: number
|
||||||
|
): Promise<void> {
|
||||||
|
// Create snapshot
|
||||||
|
const [snapshot] = await this.db
|
||||||
|
.insert(snapshots)
|
||||||
|
.values({
|
||||||
|
leagueId,
|
||||||
|
category,
|
||||||
|
status: 'success',
|
||||||
|
itemCount: data.lines.length,
|
||||||
|
})
|
||||||
|
.returning();
|
||||||
|
|
||||||
|
// Create item map from core.items (only has reference currencies)
|
||||||
|
const coreItemMap = new Map(
|
||||||
|
data.core.items.map((item) => [item.id, item])
|
||||||
|
);
|
||||||
|
|
||||||
|
let savedCount = 0;
|
||||||
|
|
||||||
|
// Process each line
|
||||||
|
for (const line of data.lines) {
|
||||||
|
// First try to get info from core.items
|
||||||
|
let itemName: string;
|
||||||
|
let itemIcon: string;
|
||||||
|
let detailsId: string;
|
||||||
|
|
||||||
|
const coreItem = coreItemMap.get(line.id);
|
||||||
|
if (coreItem) {
|
||||||
|
itemName = coreItem.name;
|
||||||
|
// Ensure full icon URL
|
||||||
|
itemIcon = coreItem.image.startsWith('http')
|
||||||
|
? coreItem.image
|
||||||
|
: `https://web.poecdn.com${coreItem.image}`;
|
||||||
|
detailsId = coreItem.detailsId;
|
||||||
|
} else {
|
||||||
|
// Get from name mapper
|
||||||
|
const info = this.mapper.getItemInfo(categoryType, line.id);
|
||||||
|
itemName = info.name;
|
||||||
|
itemIcon = info.icon; // Already full URL from search endpoint
|
||||||
|
detailsId = line.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upsert item
|
||||||
|
let item = await this.db
|
||||||
|
.select()
|
||||||
|
.from(items)
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(items.externalId, line.id),
|
||||||
|
eq(items.leagueId, leagueId)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.get();
|
||||||
|
|
||||||
|
if (!item) {
|
||||||
|
const [newItem] = await this.db
|
||||||
|
.insert(items)
|
||||||
|
.values({
|
||||||
|
externalId: line.id,
|
||||||
|
detailsId,
|
||||||
|
leagueId,
|
||||||
|
category,
|
||||||
|
name: itemName,
|
||||||
|
iconUrl: itemIcon,
|
||||||
|
})
|
||||||
|
.returning();
|
||||||
|
item = newItem;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Insert price history
|
||||||
|
await this.db.insert(priceHistory).values({
|
||||||
|
itemId: item.id,
|
||||||
|
snapshotId: snapshot.id,
|
||||||
|
divineValue: line.primaryValue,
|
||||||
|
volume: line.volumePrimaryValue,
|
||||||
|
change7d: line.sparkline?.totalChange,
|
||||||
|
sparklineData: line.sparkline?.data
|
||||||
|
? JSON.stringify(line.sparkline.data)
|
||||||
|
: null,
|
||||||
|
exaltedRate: data.core.rates.exalted,
|
||||||
|
chaosRate: data.core.rates.chaos,
|
||||||
|
});
|
||||||
|
|
||||||
|
savedCount++;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`Saved ${savedCount}/${data.lines.length} items for ${category} (snapshot #${snapshot.id})`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
206
packages/scraper/src/services/item-name-mapper.ts
Normal file
206
packages/scraper/src/services/item-name-mapper.ts
Normal file
|
|
@ -0,0 +1,206 @@
|
||||||
|
import type { PoeNinjaSearchResponse } from '@poe2data/shared';
|
||||||
|
|
||||||
|
export interface ItemNameInfo {
|
||||||
|
name: string;
|
||||||
|
icon: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Service to map poe.ninja line IDs to proper item names
|
||||||
|
* The search endpoint returns items with names, the overview returns IDs
|
||||||
|
* This service builds a mapping between them
|
||||||
|
*/
|
||||||
|
export class ItemNameMapper {
|
||||||
|
private nameMap = new Map<string, ItemNameInfo>();
|
||||||
|
private initialized = false;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize the mapper from search response data
|
||||||
|
*/
|
||||||
|
initialize(searchData: PoeNinjaSearchResponse): void {
|
||||||
|
this.nameMap.clear();
|
||||||
|
|
||||||
|
for (const [category, items] of Object.entries(searchData.items)) {
|
||||||
|
for (const item of items) {
|
||||||
|
// Generate all possible ID variations for this item
|
||||||
|
const possibleIds = this.generatePossibleIds(item.name);
|
||||||
|
|
||||||
|
for (const id of possibleIds) {
|
||||||
|
const key = `${category}:${id}`;
|
||||||
|
if (!this.nameMap.has(key)) {
|
||||||
|
this.nameMap.set(key, {
|
||||||
|
name: item.name,
|
||||||
|
icon: item.icon || '',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.initialized = true;
|
||||||
|
console.log(`ItemNameMapper initialized with ${this.nameMap.size} entries`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get item info by category and ID
|
||||||
|
*/
|
||||||
|
getItemInfo(category: string, id: string): ItemNameInfo {
|
||||||
|
const key = `${category}:${id}`;
|
||||||
|
const info = this.nameMap.get(key);
|
||||||
|
|
||||||
|
if (info) {
|
||||||
|
return info;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try without category (some items might be miscategorized)
|
||||||
|
for (const [mapKey, mapInfo] of this.nameMap) {
|
||||||
|
if (mapKey.endsWith(`:${id}`)) {
|
||||||
|
return mapInfo;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return ID as fallback
|
||||||
|
return { name: this.formatIdAsName(id), icon: '' };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if mapper is initialized
|
||||||
|
*/
|
||||||
|
isInitialized(): boolean {
|
||||||
|
return this.initialized;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate all possible ID variations from an item name
|
||||||
|
* poe.ninja uses various abbreviation schemes
|
||||||
|
*/
|
||||||
|
private generatePossibleIds(name: string): string[] {
|
||||||
|
const ids: string[] = [];
|
||||||
|
const lower = name.toLowerCase();
|
||||||
|
|
||||||
|
// Full slugified name (e.g., "divine-orb")
|
||||||
|
ids.push(this.slugify(name));
|
||||||
|
|
||||||
|
// === CURRENCY PATTERNS ===
|
||||||
|
|
||||||
|
// "Orb of X" -> shortened forms
|
||||||
|
const orbOfMatch = lower.match(/^orb of (\w+)/);
|
||||||
|
if (orbOfMatch) {
|
||||||
|
const word = orbOfMatch[1];
|
||||||
|
ids.push(word);
|
||||||
|
// Common abbreviations: first 4-5 letters
|
||||||
|
if (word.length > 4) {
|
||||||
|
ids.push(word.substring(0, 4));
|
||||||
|
ids.push(word.substring(0, 5));
|
||||||
|
}
|
||||||
|
// "alchemy" -> "alch"
|
||||||
|
if (word === 'alchemy') ids.push('alch');
|
||||||
|
if (word === 'annulment') ids.push('annul');
|
||||||
|
if (word === 'augmentation') ids.push('aug');
|
||||||
|
if (word === 'transmutation') ids.push('transmute');
|
||||||
|
if (word === 'alteration') ids.push('alt');
|
||||||
|
if (word === 'regret') ids.push('regret');
|
||||||
|
}
|
||||||
|
|
||||||
|
// "X Orb" -> first word (e.g., "Chaos Orb" -> "chaos")
|
||||||
|
const xOrbMatch = lower.match(/^(\w+)(?:'s)? orb$/);
|
||||||
|
if (xOrbMatch) {
|
||||||
|
ids.push(xOrbMatch[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// "X's Y" patterns (e.g., "Artificer's Orb" -> "artificers")
|
||||||
|
const possessiveMatch = lower.match(/^(\w+)'s (\w+)/);
|
||||||
|
if (possessiveMatch) {
|
||||||
|
ids.push(possessiveMatch[1]);
|
||||||
|
ids.push(possessiveMatch[1] + 's');
|
||||||
|
ids.push(possessiveMatch[2]);
|
||||||
|
// Combined (e.g., "gemcutters-prism")
|
||||||
|
ids.push(this.slugify(`${possessiveMatch[1]}s ${possessiveMatch[2]}`));
|
||||||
|
}
|
||||||
|
|
||||||
|
// === SPECIAL ITEM MAPPINGS ===
|
||||||
|
const specialMappings: Record<string, string[]> = {
|
||||||
|
// Currency
|
||||||
|
"gemcutter's prism": ['gcp', 'gemcutters-prism'],
|
||||||
|
"glassblower's bauble": ['bauble', 'glassblowers-bauble'],
|
||||||
|
"armourer's scrap": ['scrap', 'armourers-scrap'],
|
||||||
|
"blacksmith's whetstone": ['whetstone', 'blacksmiths-whetstone'],
|
||||||
|
"scroll of wisdom": ['wisdom', 'scroll-of-wisdom'],
|
||||||
|
"arcanist's etcher": ['etcher', 'arcanists-etcher'],
|
||||||
|
"vaal orb": ['vaal'],
|
||||||
|
"mirror of kalandra": ['mirror', 'mirror-of-kalandra'],
|
||||||
|
"regal orb": ['regal'],
|
||||||
|
"ancient orb": ['ancient'],
|
||||||
|
"fracturing orb": ['fracturing-orb'],
|
||||||
|
"hinekora's lock": ['hinekoras-lock'],
|
||||||
|
"crystallised corruption": ['crystallised-corruption'],
|
||||||
|
"vaal cultivation orb": ['vaal-cultivation-orb'],
|
||||||
|
"perfect chaos orb": ['perfect-chaos-orb'],
|
||||||
|
"perfect exalted orb": ['perfect-exalted-orb'],
|
||||||
|
"perfect jeweller's orb": ['perfect-jewellers-orb'],
|
||||||
|
"greater jeweller's orb": ['greater-jewellers-orb'],
|
||||||
|
"lesser jeweller's orb": ['lesser-jewellers-orb'],
|
||||||
|
"greater regal orb": ['greater-regal-orb'],
|
||||||
|
"lesser regal orb": ['lesser-regal-orb'],
|
||||||
|
"core destabiliser": ['core-destabiliser'],
|
||||||
|
"architect's orb": ['architects-orb'],
|
||||||
|
"ancient infuser": ['ancient-infuser'],
|
||||||
|
|
||||||
|
// Fragments
|
||||||
|
"rite of passage": ['rite-of-passage'],
|
||||||
|
"against the darkness": ['against-the-darkness'],
|
||||||
|
"the trialmaster's reliquary key": ['the-trialmasters-reliquary-key'],
|
||||||
|
"tangmazu's reliquary key": ['tangmazus-reliquary-key'],
|
||||||
|
"olroth's reliquary key": ['olroths-reliquary-key'],
|
||||||
|
|
||||||
|
// Expedition
|
||||||
|
"black scythe artifact": ['black-scythe-artifact'],
|
||||||
|
"exotic coinage": ['exotic-coinage'],
|
||||||
|
"sun artifact": ['sun-artifact'],
|
||||||
|
"broken circle artifact": ['broken-circle-artifact'],
|
||||||
|
"order artifact": ['order-artifact'],
|
||||||
|
};
|
||||||
|
|
||||||
|
if (specialMappings[lower]) {
|
||||||
|
ids.push(...specialMappings[lower]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// === GENERIC PATTERNS ===
|
||||||
|
|
||||||
|
// Hyphenated slug of full name
|
||||||
|
ids.push(this.slugify(name));
|
||||||
|
|
||||||
|
// First word if 3+ chars
|
||||||
|
const firstWord = lower.split(/[\s']+/)[0].replace(/[^a-z]/g, '');
|
||||||
|
if (firstWord.length >= 3) {
|
||||||
|
ids.push(firstWord);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove duplicates
|
||||||
|
return [...new Set(ids)];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert name to URL slug
|
||||||
|
*/
|
||||||
|
private slugify(name: string): string {
|
||||||
|
return name
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/'/g, '')
|
||||||
|
.replace(/[^a-z0-9]+/g, '-')
|
||||||
|
.replace(/^-|-$/g, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Format an ID as a readable name (fallback)
|
||||||
|
*/
|
||||||
|
private formatIdAsName(id: string): string {
|
||||||
|
return id
|
||||||
|
.split('-')
|
||||||
|
.map(word => word.charAt(0).toUpperCase() + word.slice(1))
|
||||||
|
.join(' ');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Singleton instance
|
||||||
|
export const itemNameMapper = new ItemNameMapper();
|
||||||
8
packages/scraper/tsconfig.json
Normal file
8
packages/scraper/tsconfig.json
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
{
|
||||||
|
"extends": "../../tsconfig.base.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"outDir": "dist",
|
||||||
|
"rootDir": "src"
|
||||||
|
},
|
||||||
|
"include": ["src/**/*"]
|
||||||
|
}
|
||||||
20
packages/shared/package.json
Normal file
20
packages/shared/package.json
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
{
|
||||||
|
"name": "@poe2data/shared",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"main": "./dist/index.js",
|
||||||
|
"types": "./dist/index.d.ts",
|
||||||
|
"exports": {
|
||||||
|
".": {
|
||||||
|
"import": "./dist/index.js",
|
||||||
|
"types": "./dist/index.d.ts"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"scripts": {
|
||||||
|
"build": "tsc",
|
||||||
|
"dev": "tsc --watch"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"typescript": "^5.3.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
85
packages/shared/src/constants/categories.ts
Normal file
85
packages/shared/src/constants/categories.ts
Normal file
|
|
@ -0,0 +1,85 @@
|
||||||
|
import type { CategoryConfig } from '../types/api.js';
|
||||||
|
|
||||||
|
export const POE2_CATEGORIES: CategoryConfig[] = [
|
||||||
|
{
|
||||||
|
title: 'Currency',
|
||||||
|
type: 'Currency',
|
||||||
|
url: 'currency',
|
||||||
|
icon: 'https://web.poecdn.com/gen/image/WzI1LDE0LHsiZiI6IjJESXRlbXMvQ3VycmVuY3kvQ3VycmVuY3lNb2RWYWx1ZXMiLCJzY2FsZSI6MSwicmVhbG0iOiJwb2UyIn1d/2986e220b3/CurrencyModValues.png',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Fragments',
|
||||||
|
type: 'Fragments',
|
||||||
|
url: 'fragments',
|
||||||
|
icon: 'https://web.poecdn.com/gen/image/WzI1LDE0LHsiZiI6IjJESXRlbXMvQ3VycmVuY3kvQnJlYWNoL0JyZWFjaHN0b25lIiwic2NhbGUiOjEsInJlYWxtIjoicG9lMiJ9XQ/d60587d724/Breachstone.png',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Abyssal Bones',
|
||||||
|
type: 'Abyss',
|
||||||
|
url: 'abyssal-bones',
|
||||||
|
icon: 'https://web.poecdn.com/gen/image/WzI1LDE0LHsiZiI6IjJESXRlbXMvQ3VycmVuY3kvQWJ5c3NhbEV5ZVNvY2tldGFibGVzL1RlY3JvZHNHYXplIiwic2NhbGUiOjEsInJlYWxtIjoicG9lMiJ9XQ/ef2a9355b4/TecrodsGaze.png',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Uncut Gems',
|
||||||
|
type: 'UncutGems',
|
||||||
|
url: 'uncut-gems',
|
||||||
|
icon: 'https://web.poecdn.com/gen/image/WzI1LDE0LHsiZiI6IjJESXRlbXMvR2Vtcy9VbmN1dFN1cHBvcnRHZW0iLCJzY2FsZSI6MSwicmVhbG0iOiJwb2UyIn1d/d1ffe1c951/UncutSupportGem.png',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Lineage Gems',
|
||||||
|
type: 'LineageSupportGems',
|
||||||
|
url: 'lineage-support-gems',
|
||||||
|
icon: 'https://web.poecdn.com/gen/image/WzI1LDE0LHsiZiI6IjJESXRlbXMvR2Vtcy9OZXcvTmV3U3VwcG9ydC9MaW5lYWdlL1dpbGRzaGFyZHMiLCJzY2FsZSI6MSwicmVhbG0iOiJwb2UyIn1d/6d700adf17/Wildshards.png',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Essences',
|
||||||
|
type: 'Essences',
|
||||||
|
url: 'essences',
|
||||||
|
icon: 'https://web.poecdn.com/gen/image/WzI1LDE0LHsiZiI6IjJESXRlbXMvQ3VycmVuY3kvRXNzZW5jZS9HcmVhdGVyQXR0cmlidXRlRXNzZW5jZSIsInNjYWxlIjoxLCJyZWFsbSI6InBvZTIifV0/8a8cb823af/GreaterAttributeEssence.png',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Soul Cores',
|
||||||
|
type: 'SoulCores',
|
||||||
|
url: 'soul-cores',
|
||||||
|
icon: 'https://web.poecdn.com/gen/image/WzI1LDE0LHsiZiI6IjJESXRlbXMvQ3VycmVuY3kvU291bENvcmVzL0dyZWF0ZXJTb3VsQ29yZU1hbmEiLCJzY2FsZSI6MSwicmVhbG0iOiJwb2UyIn1d/1437190de2/GreaterSoulCoreMana.png',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Idols',
|
||||||
|
type: 'Idols',
|
||||||
|
url: 'idols',
|
||||||
|
icon: 'https://web.poecdn.com/gen/image/WzI1LDE0LHsiZiI6IjJESXRlbXMvQ3VycmVuY3kvVG9ybWVudGVkU3Bpcml0U29ja2V0YWJsZXMvQXptZXJpU29ja2V0YWJsZU1vbmtleVNwZWNpYWwiLCJzY2FsZSI6MSwicmVhbG0iOiJwb2UyIn1d/8ffc9986a0/AzmeriSocketableMonkeySpecial.png',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Runes',
|
||||||
|
type: 'Runes',
|
||||||
|
url: 'runes',
|
||||||
|
icon: 'https://web.poecdn.com/gen/image/WzI1LDE0LHsiZiI6IjJESXRlbXMvQ3VycmVuY3kvUnVuZXMvTGlnaHRuaW5nUnVuZSIsInNjYWxlIjoxLCJyZWFsbSI6InBvZTIifV0/98319b3998/LightningRune.png',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Omens',
|
||||||
|
type: 'Ritual',
|
||||||
|
url: 'omens',
|
||||||
|
icon: 'https://web.poecdn.com/gen/image/WzI1LDE0LHsiZiI6IjJESXRlbXMvQ3VycmVuY3kvT21lbnMvVm9vZG9vT21lbnMzUmVkIiwic2NhbGUiOjEsInJlYWxtIjoicG9lMiJ9XQ/9cfdcc9e1a/VoodooOmens3Red.png',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Expedition',
|
||||||
|
type: 'Expedition',
|
||||||
|
url: 'expedition',
|
||||||
|
icon: 'https://web.poecdn.com/gen/image/WzI1LDE0LHsiZiI6IjJESXRlbXMvQ3VycmVuY3kvRXhwZWRpdGlvbi9CYXJ0ZXJSZWZyZXNoQ3VycmVuY3kiLCJzY2FsZSI6MSwicmVhbG0iOiJwb2UyIn1d/8a4fe1f468/BarterRefreshCurrency.png',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Liquid Emotions',
|
||||||
|
type: 'Delirium',
|
||||||
|
url: 'liquid-emotions',
|
||||||
|
icon: 'https://web.poecdn.com/gen/image/WzI1LDE0LHsiZiI6IjJESXRlbXMvQ3VycmVuY3kvRGlzdGlsbGVkRW1vdGlvbnMvRGlzdGlsbGVkUGFyYW5vaWEiLCJzY2FsZSI6MSwicmVhbG0iOiJwb2UyIn1d/279e807e8f/DistilledParanoia.png',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Catalysts',
|
||||||
|
type: 'Breach',
|
||||||
|
url: 'breach-catalyst',
|
||||||
|
icon: 'https://web.poecdn.com/gen/image/WzI1LDE0LHsiZiI6IjJESXRlbXMvQ3VycmVuY3kvQnJlYWNoL0JyZWFjaENhdGFseXN0TWFuYSIsInNjYWxlIjoxLCJyZWFsbSI6InBvZTIifV0/61d3a7a832/BreachCatalystMana.png',
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
export const CURRENT_LEAGUE = 'Fate of the Vaal';
|
||||||
|
export const LEAGUE_URL_ENCODED = 'Fate+of+the+Vaal';
|
||||||
5
packages/shared/src/index.ts
Normal file
5
packages/shared/src/index.ts
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
// Types
|
||||||
|
export * from './types/api.js';
|
||||||
|
|
||||||
|
// Constants
|
||||||
|
export * from './constants/categories.js';
|
||||||
82
packages/shared/src/types/api.ts
Normal file
82
packages/shared/src/types/api.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
||||||
|
// poe.ninja POE2 API Types
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Search endpoint response - items grouped by category
|
||||||
|
* GET /search?league={league}
|
||||||
|
*/
|
||||||
|
export interface PoeNinjaSearchResponse {
|
||||||
|
items: Record<string, PoeNinjaSearchItem[]>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PoeNinjaSearchItem {
|
||||||
|
name: string;
|
||||||
|
icon?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Overview endpoint response - category pricing data
|
||||||
|
* GET /overview?league={league}&type={type}
|
||||||
|
*/
|
||||||
|
export interface PoeNinjaOverviewResponse {
|
||||||
|
core: {
|
||||||
|
items: PoeNinjaCoreItem[];
|
||||||
|
rates: {
|
||||||
|
exalted: number;
|
||||||
|
chaos: number;
|
||||||
|
};
|
||||||
|
primary: string;
|
||||||
|
secondary: string;
|
||||||
|
};
|
||||||
|
lines: PoeNinjaLine[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PoeNinjaCoreItem {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
image: string;
|
||||||
|
category: string;
|
||||||
|
detailsId: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PoeNinjaLine {
|
||||||
|
id: string;
|
||||||
|
primaryValue: number;
|
||||||
|
volumePrimaryValue?: number;
|
||||||
|
maxVolumeCurrency?: string;
|
||||||
|
maxVolumeRate?: number;
|
||||||
|
sparkline?: {
|
||||||
|
totalChange: number;
|
||||||
|
data: number[];
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Details endpoint response - individual item details
|
||||||
|
* GET /details?league={league}&type={type}&id={id}
|
||||||
|
*/
|
||||||
|
export interface PoeNinjaDetailsResponse {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
type: string;
|
||||||
|
icon?: string;
|
||||||
|
tradeId?: string;
|
||||||
|
chaosValue?: number;
|
||||||
|
divineValue?: number;
|
||||||
|
exaltedValue?: number;
|
||||||
|
sparkline?: number[];
|
||||||
|
history?: PriceHistoryPoint[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PriceHistoryPoint {
|
||||||
|
date: string;
|
||||||
|
value: number;
|
||||||
|
volume?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Category configuration
|
||||||
|
export interface CategoryConfig {
|
||||||
|
title: string;
|
||||||
|
type: string;
|
||||||
|
url: string;
|
||||||
|
icon: string;
|
||||||
|
}
|
||||||
8
packages/shared/tsconfig.json
Normal file
8
packages/shared/tsconfig.json
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
{
|
||||||
|
"extends": "../../tsconfig.base.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"outDir": "dist",
|
||||||
|
"rootDir": "src"
|
||||||
|
},
|
||||||
|
"include": ["src/**/*"]
|
||||||
|
}
|
||||||
3913
pnpm-lock.yaml
generated
Normal file
3913
pnpm-lock.yaml
generated
Normal file
File diff suppressed because it is too large
Load diff
3
pnpm-workspace.yaml
Normal file
3
pnpm-workspace.yaml
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
packages:
|
||||||
|
- 'packages/*'
|
||||||
|
- 'apps/*'
|
||||||
18
tsconfig.base.json
Normal file
18
tsconfig.base.json
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ES2022",
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"esModuleInterop": true,
|
||||||
|
"strict": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"declaration": true,
|
||||||
|
"declarationMap": true,
|
||||||
|
"sourceMap": true,
|
||||||
|
"outDir": "dist",
|
||||||
|
"rootDir": "src",
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"isolatedModules": true,
|
||||||
|
"noEmit": false
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue