boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

removed migration files

Browse source
This commit is contained in:
Boki 2025-06-22 18:43:49 -04:00
parent 80c1dcb6cb
commit 9a5e87ef4a
16 changed files with 257 additions and 207 deletions
Download patch file Download diff file Expand all files Collapse all files

8
apps/data-pipeline/src/clients.ts
View file

@ -1,8 +0,0 @@
/**
* Client exports for backward compatibility
*
* @deprecated Use ServiceContainer parameter instead
* This file will be removed once all operations are migrated
*/
export { getMongoDBClient, getPostgreSQLClient } from './migration-helper';

1
apps/data-pipeline/src/handlers/exchanges/operations/qm-exchanges.operations.ts
View file

@ -1,5 +1,4 @@
import { getLogger } from '@stock-bot/logger';
import { getMongoDBClient, getPostgreSQLClient } from '../../../clients';
import type { IServiceContainer } from '@stock-bot/handlers';
import type { JobPayload } from '../../../types/job-payloads';

1
apps/data-pipeline/src/handlers/exchanges/operations/sync-ib-exchanges.operations.ts
View file

@ -1,6 +1,5 @@
import { getLogger } from '@stock-bot/logger';
import type { MasterExchange } from '@stock-bot/mongodb';
import { getMongoDBClient } from '../../../clients';
import type { IServiceContainer } from '@stock-bot/handlers';
import type { JobPayload } from '../../../types/job-payloads';

1
apps/data-pipeline/src/handlers/exchanges/operations/sync-qm-provider-mappings.operations.ts
View file

@ -1,5 +1,4 @@
import { getLogger } from '@stock-bot/logger';
import { getMongoDBClient, getPostgreSQLClient } from '../../../clients';
import type { IServiceContainer } from '@stock-bot/handlers';
import type { JobPayload, SyncResult } from '../../../types/job-payloads';

9
apps/data-pipeline/src/handlers/symbols/operations/qm-symbols.operations.ts
View file

@ -1,17 +1,18 @@
import { getLogger } from '@stock-bot/logger';
import { getMongoDBClient, getPostgreSQLClient } from '../../../clients';
import type { IServiceContainer } from '@stock-bot/handlers';
import type { JobPayload } from '../../../types/job-payloads';
const logger = getLogger('sync-qm-symbols');
export async function syncQMSymbols(
payload: JobPayload
payload: JobPayload,
container: IServiceContainer
): Promise<{ processed: number; created: number; updated: number }> {
logger.info('Starting QM symbols sync...');
try {
const mongoClient = getMongoDBClient();
const postgresClient = getPostgreSQLClient();
const mongoClient = container.mongodb;
const postgresClient = container.postgres;
// 1. Get all QM symbols from MongoDB
const qmSymbols = await mongoClient.find('qmSymbols', {});

9
apps/data-pipeline/src/handlers/symbols/operations/sync-status.operations.ts
View file

@ -1,14 +1,17 @@
import { getLogger } from '@stock-bot/logger';
import { getPostgreSQLClient } from '../../../clients';
import type { IServiceContainer } from '@stock-bot/handlers';
import type { JobPayload } from '../../../types/job-payloads';
const logger = getLogger('sync-status');
export async function getSyncStatus(payload: JobPayload): Promise<Record<string, unknown>[]> {
export async function getSyncStatus(
payload: JobPayload,
container: IServiceContainer
): Promise<Record<string, unknown>[]> {
logger.info('Getting sync status...');
try {
const postgresClient = getPostgreSQLClient();
const postgresClient = container.postgres;
const query = 'SELECT * FROM sync_status ORDER BY provider, data_type';
const result = await postgresClient.query(query);

56
apps/data-pipeline/src/handlers/symbols/operations/sync-symbols-from-provider.operations.ts
View file

@ -1,10 +1,13 @@
import { getLogger } from '@stock-bot/logger';
import { getMongoDBClient, getPostgreSQLClient } from '../../../clients';
import type { IServiceContainer } from '@stock-bot/handlers';
import type { JobPayload, SyncResult } from '../../../types/job-payloads';
const logger = getLogger('enhanced-sync-symbols-from-provider');
export async function syncSymbolsFromProvider(payload: JobPayload): Promise<SyncResult> {
export async function syncSymbolsFromProvider(
payload: JobPayload,
container: IServiceContainer
): Promise<SyncResult> {
const provider = payload.provider;
const clearFirst = payload.clearFirst || false;
@ -23,8 +26,8 @@ export async function syncSymbolsFromProvider(payload: JobPayload): Promise<Sync
};
try {
const mongoClient = getMongoDBClient();
const postgresClient = getPostgreSQLClient();
const mongoClient = container.mongodb;
const postgresClient = container.postgres;
// Clear existing data if requested (only symbols and mappings, keep exchanges)
if (clearFirst) {
@ -61,7 +64,7 @@ export async function syncSymbolsFromProvider(payload: JobPayload): Promise<Sync
for (const symbol of symbols) {
try {
await processSingleSymbol(symbol, provider, result);
await processSingleSymbol(symbol, provider, result, container);
} catch (error) {
logger.error('Failed to process symbol', {
error,
@ -73,15 +76,14 @@ export async function syncSymbolsFromProvider(payload: JobPayload): Promise<Sync
}
// Update sync status
await updateSyncStatus(provider, 'symbols', result.processed, postgresClient);
await updateSyncStatus(provider, 'symbols', result.processed, container.postgres);
await postgresClient.query('COMMIT');
logger.info(`${provider} symbols sync completed`, result);
return result;
} catch (error) {
const postgresClient = getPostgreSQLClient();
await postgresClient.query('ROLLBACK');
await container.postgres.query('ROLLBACK');
logger.error(`${provider} symbols sync failed`, { error });
throw error;
}
@ -90,7 +92,8 @@ export async function syncSymbolsFromProvider(payload: JobPayload): Promise<Sync
async function processSingleSymbol(
symbol: any,
provider: string,
result: SyncResult
result: SyncResult,
container: IServiceContainer
): Promise<void> {
const symbolCode = symbol.symbol || symbol.code;
const exchangeCode = symbol.exchangeCode || symbol.exchange || symbol.exchange_id;
@ -101,7 +104,7 @@ async function processSingleSymbol(
}
// Find active provider exchange mapping
const providerMapping = await findActiveProviderExchangeMapping(provider, exchangeCode);
const providerMapping = await findActiveProviderExchangeMapping(provider, exchangeCode, container);
if (!providerMapping) {
result.skipped++;
@ -111,25 +114,27 @@ async function processSingleSymbol(
// Check if symbol exists
const existingSymbol = await findSymbolByCodeAndExchange(
symbolCode,
providerMapping.master_exchange_id
providerMapping.master_exchange_id,
container
);
if (existingSymbol) {
await updateSymbol(existingSymbol.id, symbol);
await upsertProviderMapping(existingSymbol.id, provider, symbol);
await updateSymbol(existingSymbol.id, symbol, container);
await upsertProviderMapping(existingSymbol.id, provider, symbol, container);
result.updated++;
} else {
const newSymbolId = await createSymbol(symbol, providerMapping.master_exchange_id);
await upsertProviderMapping(newSymbolId, provider, symbol);
const newSymbolId = await createSymbol(symbol, providerMapping.master_exchange_id, container);
await upsertProviderMapping(newSymbolId, provider, symbol, container);
result.created++;
}
}
async function findActiveProviderExchangeMapping(
provider: string,
providerExchangeCode: string
providerExchangeCode: string,
container: IServiceContainer
): Promise<any> {
const postgresClient = getPostgreSQLClient();
const postgresClient = container.postgres;
const query = `
SELECT pem.*, e.code as master_exchange_code
FROM provider_exchange_mappings pem
@ -140,15 +145,15 @@ async function findActiveProviderExchangeMapping(
return result.rows[0] || null;
}
async function findSymbolByCodeAndExchange(symbol: string, exchangeId: string): Promise<any> {
const postgresClient = getPostgreSQLClient();
async function findSymbolByCodeAndExchange(symbol: string, exchangeId: string, container: IServiceContainer): Promise<any> {
const postgresClient = container.postgres;
const query = 'SELECT * FROM symbols WHERE symbol = $1 AND exchange_id = $2';
const result = await postgresClient.query(query, [symbol, exchangeId]);
return result.rows[0] || null;
}
async function createSymbol(symbol: any, exchangeId: string): Promise<string> {
const postgresClient = getPostgreSQLClient();
async function createSymbol(symbol: any, exchangeId: string, container: IServiceContainer): Promise<string> {
const postgresClient = container.postgres;
const query = `
INSERT INTO symbols (symbol, exchange_id, company_name, country, currency)
VALUES ($1, $2, $3, $4, $5)
@ -166,8 +171,8 @@ async function createSymbol(symbol: any, exchangeId: string): Promise<string> {
return result.rows[0].id;
}
async function updateSymbol(symbolId: string, symbol: any): Promise<void> {
const postgresClient = getPostgreSQLClient();
async function updateSymbol(symbolId: string, symbol: any, container: IServiceContainer): Promise<void> {
const postgresClient = container.postgres;
const query = `
UPDATE symbols
SET company_name = COALESCE($2, company_name),
@ -188,9 +193,10 @@ async function updateSymbol(symbolId: string, symbol: any): Promise<void> {
async function upsertProviderMapping(
symbolId: string,
provider: string,
symbol: any
symbol: any,
container: IServiceContainer
): Promise<void> {
const postgresClient = getPostgreSQLClient();
const postgresClient = container.postgres;
const query = `
INSERT INTO provider_mappings
(symbol_id, provider, provider_symbol, provider_exchange, last_seen)

4
apps/data-pipeline/src/index.ts
View file

@ -71,10 +71,6 @@ async function initializeServices() {
// Setup service-specific configuration
const serviceContainer = setupServiceContainer(config, container.resolve('serviceContainer'));
// Initialize migration helper for backward compatibility
const { setContainerForMigration } = await import('./migration-helper');
setContainerForMigration(serviceContainer);
logger.info('Migration helper initialized for backward compatibility');
// Create app with routes
app = new Hono();

37
apps/data-pipeline/src/migration-helper.ts
View file

@ -1,37 +0,0 @@
/**
* Temporary migration helper for data-pipeline service
* Provides backward compatibility while migrating to DI container
*
* TODO: Remove this file once all operations are migrated to use ServiceContainer
*/
import type { ServiceContainer } from '@stock-bot/di';
import type { MongoDBClient } from '@stock-bot/mongodb';
import type { PostgreSQLClient } from '@stock-bot/postgres';
let containerInstance: ServiceContainer | null = null;
export function setContainerForMigration(container: ServiceContainer): void {
containerInstance = container;
}
export function getMongoDBClient(): MongoDBClient {
if (!containerInstance) {
throw new Error('Container not initialized. This is a migration helper - please update the operation to accept ServiceContainer parameter');
}
return containerInstance.mongodb;
}
export function getPostgreSQLClient(): PostgreSQLClient {
if (!containerInstance) {
throw new Error('Container not initialized. This is a migration helper - please update the operation to accept ServiceContainer parameter');
}
return containerInstance.postgres;
}
export function getQuestDBClient(): any {
if (!containerInstance) {
throw new Error('Container not initialized. This is a migration helper - please update the operation to accept ServiceContainer parameter');
}
return containerInstance.questdb;
}