From 7d9044ab2904cecdf7556d5ef65d24a6a2d6e420 Mon Sep 17 00:00:00 2001 From: Boki Date: Sun, 22 Jun 2025 17:55:51 -0400 Subject: [PATCH] format --- .../document_symbols_cache_v20-05-25.pkl | Bin 0 -> 102226 bytes .serena/memories/code_style_conventions.md | 58 ++ .../memories/current_refactoring_context.md | 41 + .serena/memories/project_overview.md | 55 ++ .serena/memories/project_structure.md | 62 ++ .serena/memories/suggested_commands.md | 73 ++ .serena/memories/task_completion_checklist.md | 55 ++ .serena/memories/tech_stack.md | 49 ++ .serena/project.yml | 66 ++ .vscode/mcp.json | 20 +- CLAUDE.md | 171 ---- MIGRATION-TO-CONNECTION-POOLS.md | 183 ----- apps/data-ingestion/config/default.json | 2 +- .../src/handlers/ceo/actions/index.ts | 6 +- .../process-individual-symbol.action.ts | 228 +++--- .../ceo/actions/update-ceo-channels.action.ts | 139 ++-- .../actions/update-unique-symbols.action.ts | 134 ++-- .../src/handlers/ceo/ceo.handler.ts | 26 +- .../src/handlers/example-handler.ts | 221 +++--- .../src/handlers/example/example.handler.ts | 188 ++--- .../fetch-exchanges-and-symbols.action.ts | 38 + .../fetch-exchanges.action.ts} | 38 +- .../ib/actions/fetch-session.action.ts | 83 ++ .../fetch-symbols.action.ts} | 83 +- .../src/handlers/ib/actions/index.ts | 5 + .../src/handlers/ib/ib.handler.ts | 109 +-- .../ib/operations/session.operations.ts | 91 --- .../src/handlers/ib/shared/config.ts | 7 +- apps/data-ingestion/src/handlers/index.ts | 32 +- .../proxy/operations/check.operations.ts | 34 +- .../proxy/operations/query.operations.ts | 12 +- .../proxy/operations/queue.operations.ts | 8 +- .../src/handlers/proxy/proxy.handler.ts | 10 +- .../src/handlers/proxy/shared/config.ts | 2 +- .../src/handlers/proxy/shared/types.ts | 2 +- .../handlers/qm/actions/exchanges.action.ts | 12 +- .../src/handlers/qm/actions/session.action.ts | 30 +- .../src/handlers/qm/actions/spider.action.ts | 9 +- .../src/handlers/qm/actions/symbols.action.ts | 12 +- .../src/handlers/qm/qm.handler.ts | 45 +- .../src/handlers/qm/shared/config.ts | 5 +- .../src/handlers/qm/shared/session-manager.ts | 36 +- .../src/handlers/qm/shared/types.ts | 2 +- .../webshare/operations/fetch.operations.ts | 55 +- .../src/handlers/webshare/shared/config.ts | 2 +- .../src/handlers/webshare/webshare.handler.ts | 39 +- apps/data-ingestion/src/index.ts | 44 +- .../src/routes/create-routes.ts | 143 ++-- .../src/routes/exchange.routes.ts | 4 +- .../data-ingestion/src/routes/queue.routes.ts | 6 +- .../src/types/exchange.types.ts | 2 +- apps/data-ingestion/src/types/job-payloads.ts | 2 +- .../src/utils/symbol-search.util.ts | 2 +- apps/data-ingestion/test-ceo-operations.ts | 204 ++--- apps/data-pipeline/config/default.json | 30 +- apps/data-pipeline/src/clients.ts | 54 +- .../handlers/exchanges/exchanges.handler.ts | 116 +-- .../clear-postgresql-data.operations.ts | 8 +- .../enhanced-sync-status.operations.ts | 4 +- .../operations/exchange-stats.operations.ts | 4 +- .../handlers/exchanges/operations/index.ts | 38 +- .../provider-mapping-stats.operations.ts | 4 +- .../operations/qm-exchanges.operations.ts | 215 ++--- .../sync-all-exchanges.operations.ts | 541 ++++++------- .../sync-ib-exchanges.operations.ts | 414 +++++----- .../sync-qm-provider-mappings.operations.ts | 410 +++++----- .../src/handlers/symbols/operations/index.ts | 18 +- .../operations/qm-symbols.operations.ts | 350 ++++---- .../operations/sync-status.operations.ts | 42 +- .../sync-symbols-from-provider.operations.ts | 446 ++++++----- .../src/handlers/symbols/symbols.handler.ts | 82 +- apps/data-pipeline/src/index.ts | 60 +- .../src/routes/enhanced-sync.routes.ts | 24 +- apps/data-pipeline/src/routes/index.ts | 2 +- apps/data-pipeline/src/routes/stats.routes.ts | 10 +- apps/data-pipeline/src/routes/sync.routes.ts | 18 +- apps/data-pipeline/src/types/job-payloads.ts | 54 +- apps/web-api/config/default.json | 30 +- apps/web-api/src/clients.ts | 54 +- apps/web-api/src/index.ts | 48 +- apps/web-api/src/routes/exchange.routes.ts | 116 ++- apps/web-api/src/routes/health.routes.ts | 16 +- apps/web-api/src/services/exchange.service.ts | 47 +- apps/web-api/src/types/exchange.types.ts | 2 +- apps/web-api/src/utils/error-handler.ts | 4 +- apps/web-api/src/utils/validation.ts | 12 +- .../features/exchanges/hooks/useExchanges.ts | 29 +- .../exchanges/hooks/useFormValidation.ts | 67 +- .../exchanges/services/exchangeApi.ts | 23 +- .../src/features/exchanges/types/api.types.ts | 2 +- .../exchanges/types/component.types.ts | 6 +- .../features/exchanges/types/request.types.ts | 2 +- .../features/exchanges/utils/formatters.ts | 4 +- .../features/exchanges/utils/validation.ts | 2 +- apps/web-app/src/lib/utils.ts | 8 +- apps/web-app/src/lib/utils/index.ts | 24 +- docs/enhanced-cache-usage.md | 148 ---- docs/loki-logging.md | 169 ---- docs/mongodb-multi-database-migration.md | 212 ----- libs/core/config/config/default.json | 2 +- libs/core/config/config/development.json | 2 +- libs/core/config/config/production.json | 2 +- libs/core/config/config/test.json | 2 +- libs/core/config/src/cli.ts | 389 +++++---- libs/core/config/src/errors.ts | 12 +- libs/core/config/src/schemas/base.schema.ts | 2 +- .../config/src/schemas/database.schema.ts | 2 +- libs/core/config/src/schemas/index.ts | 168 ++-- .../config/src/schemas/provider.schema.ts | 12 +- .../core/config/src/schemas/service.schema.ts | 70 +- libs/core/config/src/utils/secrets.ts | 361 +++++---- libs/core/config/src/utils/validation.ts | 383 +++++---- libs/core/config/test/config-manager.test.ts | 436 +++++----- .../core/config/test/dynamic-location.test.ts | 144 ++-- libs/core/config/test/edge-cases.test.ts | 129 ++- libs/core/config/test/index.test.ts | 410 +++++----- libs/core/config/test/loaders.test.ts | 347 ++++---- libs/core/config/test/provider-config.test.ts | 87 +- libs/core/config/test/real-usage.test.ts | 185 +++-- libs/core/config/tsconfig.json | 5 +- libs/core/di/package.json | 34 +- libs/core/di/src/awilix-container.ts | 595 +++++++------- libs/core/di/src/index.ts | 26 +- libs/core/di/src/operation-context.ts | 23 +- libs/core/di/src/pool-size-calculator.ts | 162 ++-- libs/core/di/src/types.ts | 139 ++-- libs/core/di/test/di.test.ts | 361 ++++----- libs/core/di/tsconfig.json | 31 +- libs/core/handlers/package.json | 46 +- libs/core/handlers/src/base/BaseHandler.ts | 604 +++++++------- .../handlers/src/decorators/decorators.ts | 278 +++---- libs/core/handlers/src/index.ts | 68 +- .../handlers/src/registry/HandlerRegistry.ts | 384 ++++----- .../handlers/src/registry/auto-register.ts | 368 ++++----- .../handlers/src/types/service-container.ts | 54 +- libs/core/handlers/src/types/types.ts | 28 +- libs/core/handlers/tsconfig.json | 30 +- libs/core/logger/src/logger.ts | 7 +- libs/core/logger/tsconfig.json | 3 +- libs/core/types/src/backtesting.ts | 6 +- libs/core/types/src/financial-statements.ts | 38 +- libs/core/types/src/handler-registry.ts | 230 +++--- libs/core/types/src/handlers.ts | 166 ++-- libs/core/types/src/helpers.ts | 2 +- libs/core/types/src/market-data.ts | 2 +- libs/core/types/src/options.ts | 2 +- libs/core/types/src/portfolio.ts | 2 +- libs/core/types/src/risk-metrics.ts | 2 +- libs/core/types/src/technical-analysis.ts | 24 +- libs/core/types/src/trading.ts | 2 +- libs/core/types/tsconfig.json | 3 +- libs/data/cache/src/index.ts | 8 +- libs/data/cache/tsconfig.json | 4 +- libs/data/mongodb/src/client.ts | 66 +- libs/data/mongodb/src/index.ts | 22 +- libs/data/mongodb/tsconfig.json | 5 +- libs/data/postgres/src/client.ts | 61 +- libs/data/postgres/tsconfig.json | 5 +- libs/data/questdb/src/client.ts | 17 +- libs/data/questdb/src/query-builder.ts | 6 +- libs/data/questdb/src/schema.ts | 1 - libs/data/questdb/tsconfig.json | 5 +- libs/services/browser/src/browser.ts | 6 +- libs/services/browser/tsconfig.json | 4 +- libs/services/event-bus/src/event-bus.ts | 30 +- libs/services/event-bus/src/index.ts | 2 +- libs/services/event-bus/src/types.ts | 12 +- libs/services/event-bus/tsconfig.json | 4 +- libs/services/proxy/package.json | 50 +- libs/services/proxy/src/index.ts | 35 +- libs/services/proxy/src/proxy-manager.ts | 571 +++++++------- libs/services/proxy/src/types.ts | 84 +- libs/services/proxy/tsconfig.json | 24 +- libs/services/queue/src/dlq-handler.ts | 500 ++++++------ libs/services/queue/src/index.ts | 13 +- libs/services/queue/src/queue-manager.ts | 13 +- libs/services/queue/src/queue-metrics.ts | 632 +++++++-------- libs/services/queue/src/queue.ts | 744 +++++++++--------- libs/services/queue/src/rate-limiter.ts | 621 ++++++++------- libs/services/queue/src/types.ts | 14 +- libs/services/queue/src/utils.ts | 2 +- .../queue/test/batch-processor.test.ts | 719 ++++++++--------- libs/services/queue/test/dlq-handler.test.ts | 736 ++++++++--------- .../queue/test/queue-integration.test.ts | 18 +- .../services/queue/test/queue-manager.test.ts | 742 ++++++++--------- .../services/queue/test/queue-metrics.test.ts | 630 ++++++++------- libs/services/queue/test/queue-simple.test.ts | 162 ++-- libs/services/queue/test/rate-limiter.test.ts | 620 +++++++-------- libs/services/shutdown/src/index.ts | 13 +- libs/services/shutdown/src/shutdown.ts | 7 +- libs/services/shutdown/tsconfig.json | 3 +- libs/utils/package.json | 2 +- libs/utils/src/calculations/index.ts | 28 +- .../src/calculations/performance-metrics.ts | 101 ++- libs/utils/src/calculations/risk-metrics.ts | 35 +- .../src/calculations/technical-indicators.ts | 80 +- libs/utils/src/fetch.ts | 190 +++-- libs/utils/src/generic-functions.ts | 42 +- libs/utils/src/user-agent.ts | 60 +- scripts/setup-mcp.sh | 83 -- tsconfig.app.json | 37 +- tsconfig.lib.json | 34 +- 202 files changed, 10755 insertions(+), 10972 deletions(-) create mode 100644 .serena/cache/typescript/document_symbols_cache_v20-05-25.pkl create mode 100644 .serena/memories/code_style_conventions.md create mode 100644 .serena/memories/current_refactoring_context.md create mode 100644 .serena/memories/project_overview.md create mode 100644 .serena/memories/project_structure.md create mode 100644 .serena/memories/suggested_commands.md create mode 100644 .serena/memories/task_completion_checklist.md create mode 100644 .serena/memories/tech_stack.md create mode 100644 .serena/project.yml delete mode 100644 CLAUDE.md delete mode 100644 MIGRATION-TO-CONNECTION-POOLS.md create mode 100644 apps/data-ingestion/src/handlers/ib/actions/fetch-exchanges-and-symbols.action.ts rename apps/data-ingestion/src/handlers/ib/{operations/exchanges.operations.ts => actions/fetch-exchanges.action.ts} (55%) create mode 100644 apps/data-ingestion/src/handlers/ib/actions/fetch-session.action.ts rename apps/data-ingestion/src/handlers/ib/{operations/symbols.operations.ts => actions/fetch-symbols.action.ts} (58%) create mode 100644 apps/data-ingestion/src/handlers/ib/actions/index.ts delete mode 100644 apps/data-ingestion/src/handlers/ib/operations/session.operations.ts delete mode 100644 docs/enhanced-cache-usage.md delete mode 100644 docs/loki-logging.md delete mode 100644 docs/mongodb-multi-database-migration.md delete mode 100755 scripts/setup-mcp.sh diff --git a/.serena/cache/typescript/document_symbols_cache_v20-05-25.pkl b/.serena/cache/typescript/document_symbols_cache_v20-05-25.pkl new file mode 100644 index 0000000000000000000000000000000000000000..44c37799bcb459ebc02e092679ca96386b449943 GIT binary patch literal 102226 zcmd6Q3!EHPm3JWXN@gPn}3=m#XO zipZh@y{mjQ-@2~;__4Y^KwV!eqPyy{_!eEiZ+W<@tBCvUuA=MbhmUnx{Lblny1MJm zJyq2c&Q197JCpubovK^^d+xdCo_p)oo4W2fao!2@@V__IH`T`SxpX;^DVCgsn<`}U zPN|$Imoka016Zn3grBW{ZmpEJT$IX}oqBEk_HElz>CtRz$H?&5==ey=nbTbl%CUc2;# zyvsnz$-~p!NZ0PTnAKX(ObVExN>DYpuw@6$daJEERLxg%`SQ$>t^U8RuN;|i%Jtg9 z{YMU^i}@s0@ZI&~3Afd+uDff@6+Dlv;H}BJn@DyUD%d^8U5PciTB4%dHE{X0FKDf} z>uPhTh;}R8K_kg+2^UZyUY9H=W?Tl7~!1GV3 zR^1f`>avW!_yuRwU0-!KxTjYYEeo!galmI`D>f2yN`RQn24e7x7BTp1l_EynVBxN* zYN6p{&Nts-C1@=0sXrTj%?JH<>dj{W_2?|2YWp+ALb+0^X2A6}?=cvt-qYZ!3;lcG zug%&$W}k5~RqqD(a@i_=g+vuo_4XWcDsC~`WTiL2%2~iekIxGJ0v{h>e)bsywpeil zSyBBf*?W+aGXtF9uZ0>X<}=U{h+E1HBAhr-@V&Aoex*pkQ?iA=SfbEvu6NN@Rj2Cg z&$v#uns=H_G}tc@?l%d)boXmMqvd{k zi!<1Nl$%cwY)}`&61QO)fEXTzCVtovw3>^6jF#2luO*t*n9s0cH6IbAW8!<+3zzSQP)M5Cg zcT#vptCPZCr;|;J4oANs#t_&f_OEBu+DG{AS?;D}S7XdhkG1CoNQbBdfO{_9Z)Byq4r-@c+6c#lMAWCID126UZ!4k$05o+J=J{n`H=Lg!$UvWY@8EwecX7@INb3w z;)AU)KbU%&P3~i=9=p;hNYS-F%z1Ntn1s3%rfs5XZF|R0L>FIIJJtMH>Ug83)Pwc! zh6nR=A7{Cn8~G$-!6wf>0jvw#?uh%FW}RMkR5T>o?hctysAstH+S=Z^J@>4ut*dPg zp+P^r!Ag6AHNcIZV|DwQDc0_0?h&Xoq*(j844l^o$>PZsCwL3sFPCZcdf*^jxN&l_ zU!A+6>Mb0s`|I}sqa&zlc_k_)^px>DdqE7gro)Nb@51w-ub4a`k# zF{mBDRB%+m2F(RKQXOgtKgK~--|um z@)!8wyp3?dDh`yCvGk@(#hc17Q4z44`7I;swP;m~& zYwfQ!^O}p`mDF@z35LW1^+2s4xesQ+QrT1`6|8P1zR0Ni_h|Jtt>9j`zA-{()Ce9t z>SATp{rj|H5zK}?bX@PR+tmE{`1NQjl(~4Jhkf2#g$;2*C3nbyRWOHUVpXzZW|Lr; zLNQlzxEcmccG$dIn9be_tQ>OoWn*zUW#*FLEXQTI8YHbPae1!{myja#%91VWjWXr- z2W4HjC@Gc!q?G~a9+P00KsPt0Xf?kvi4yry86t7sLT*T2l_|7e`@K%1OV7K4Hi0)kMe0- z<c5ic!15=MMbjqmq8|N#T~FRugqVZvy#>S zdztDNH``y!l-oDkz{0A4+ht~!wkUJeXO;+NXO_a%0y9gZZeZ}l7j|Z8*5nJl#1afp znB>TaIgTQD9oMG9)fV|pU)rUv7Ut3zRxxI|+iaEuZ|A5CS8Jbuk?oycbS$W3ta6{( zDhbAvpfP0^t_DlpTNwz^7Cs_OW!s{{fWKte+-7EzU>uei*^E~6`%O_c&p*D{yu{2V z!7Q86YIfMXRG7_0{(yNBlJvnoh;f_~X2(hJBxj%DYVbCf9)7O&i&ow`1+V0FI+&*$ z>Z5XkFN-K=h?-yX*I?yHW$dq)sd@4E^JOyS_T$fmEwhMau3OFKO7IwG2H|SUeRGu# znF_*mE-g7XRAJ`EXNXd0ZcDlUn<*S#Wj)_G@@SS7Yj9=7nymyx%F^n9FEfnvFC0 zh?&c;D43lfAvMi1n^$WsA@M7@7Zy>@E))|-eM#M2uPK7hGmgU&rC3z^rGV{c{M+f|Dt3=`G&B~_2i4w(=ius z6DQluoLt0JaJbqIP_skj6Vg=DehnR1< zpARu`Fg{}(`2A*I`GA7iro^Zbyuh#jIr>Vv{mGhD63qHaI`>V|Up59e&b2TpZQ?|CdIWEcppsDw_)1Z$TqZ-MxUc-B zaBJ#xl5%MY8KUFE2u3y4mgeXzKia%~@U2gRF_<>aw9xTInqz2F^IQ0O zv>d4s*f}z0>34-)>0x0 zo4F+Ta*oSzwN_f7)l?Sjsum6wiZ>O`@1$HiG^?lkvePMaJVecJEEh;Nmdk|ecyMFm z-J3#X%QdiWdt)b*o@}OcJff-@wKG7?4y7-bp%lAC(dNEKrrf^Y>?Fwigt@g7%)ST{ zt`=B@B1+}7G?g6;$p6wzCBd+_qPaH@SF=ZDS%ymSAoFyYa{H}m{!BGqjQJ^5|9bNB zb4{~g>Yq^~_^1cupKjhh7-JI5#+Y{WNtN^JP^G@N~jUQuv#!Mu^Y>XMM z7KkxLiTo2`B0GvPA2M@EFdJirtGxg*Cfz^}771o!%xE>gvD_!wSUxOV$8(7>i5opw zB$$mc!_@*YrYNPKk)afOMbYN|UYTyaDa5Z~W{%;v7#beAjiBNAr~vJHMx}GvWdg)w8j?hE9zV} z72A^^g1&N#Fqgf>!v4nR*8;1y%(2kqGKS11&b_SYE9zu663Hc?E+_1=eI?6jI8 zv0xLwN|?#+nNsm^yrl?>%`_5>-G?z6qT@P^1htZR2rs*PVRK z$mC=X^+IeO!>rf5oM{nR4}8&ZDyJn^n74R-txBc8#q&D_cXR%RQ6sojzir8t#y3sl z^d!Gg_tqyu31*X&;c6|OAR+KJGz_Wa%H6_N$G$+!wBX;G=_DAZLX161xLT`*rRls^ zn9fCI`h;E<-WZJ+>inmfQG(g7C0eZ+>c|@C12T+a7X4<)hINNbx&5S7$2&tkuIxbf zodhSimKCnn{)($CpPv=x^HjCDH#95yfKaD9VEnV}|)H4)X`>8)8?wgc z&t%xdZs$=+D|k|-+`jjTzftxN&25}uHZTrXYfo0m68W4Ak!S#qNE*PmWy*~^8HvVA zUzRDiZ@B!HAWYZ!gxP)x-pln-;cBg}Q<~4gZrRB`o=>KGeO~dY50nXJb2HIue)lSB zpR0xWJW-zqV9fISipG<)ewARB#%MJ=G`>if#wGrSWfaD1gJg!)@0lAr!DC!w4_9mL z@+{rhM}%2rvp5W$Pnzi@n58pZt+n5ZG@a)kD>@n1`FpeLB=}m+I>XgkNin~VJjxF` z(Eao`$O*nCqL?9S$~Co(>N;3NN_W>+u@QV#M4=cp+;GD8msg>+Nabf=Eo_SMKj=V= z4Qm|_!R(klTn$};9U@&BA~Bx7MA88E%amI@p1)M4+`g|0%Y-w%(!ZH|rDvPomB~6W zY6RmzR#~uLw3W>y{hU6FL)YD;wKFOuJI58RrhC!;OsQD7GF5>o!9uJ>6F*l(X$wh` z1mhxBW7aNQt$opicZ+08stH?Uf7U5yO1T;QG-ND3|F4-(g7YPZ0c_$5d1QZ&2Y7oK~3|PN4xVl z-B9*uRT<(~rrN+wVj!(H)mq!-N)!2v43RkgJdBr?Ne~06z$@vVYO`PSld4gUU zP23)*YlJ!NEGnHmfnl_XujkrO^8}sICLV}PMXhv`45Qd@Jtx_3^~#i6yx)3WxZhgP z>=@!rjDA;jS^7#ng10gDn+#VA_yf^d`<*gY0M;w`X3{OfOJNo^O<X)JP3NxjzjG&a%v|b@e z@G5AHdSKNJA0$b|8>zlR-F&lFUj(xY#G}>x6v}?A45@+2YlLgM*DIzkOzB&$Fv-(v z6qmZ&e7StIe4;s6YHCz_1;m(4)pXtez+wUDsukJD) zbHB0x-D46w&RIaXS|I5x%H+Fbn3N4PuazpaR3G0h@=Z<%=6_Z7ZFceg<|~$zT?x+R zcY9q>2t2!o{`SG&$u%_6pBIdtCN~j&0%O3wqW0R6kG}{Ml@n`~Q2TPIHj*=koRb@< zBw83hxsh-#C}9z+}g z7yL^{>h4+Ylg($kU@Dh)b|w;?XBpLg0eHDJc!P6MY)#H{6Ulj1cc}eW71EeHkLCvJ zYU`L(r(cMwJAR;!aW6dSNz^j0ZtQ5F9(Z#|DZc<^hLwiyi1)2F^StJ$nMr8nXyjqn zsGF>uz7I>mYply`CJb zb(d0wX{Qc+*SgD%4WICzUHM$WsV6^QTaa;6rBnuYlSnSA*5*5)1t)kv0DhC7_kM_f zU`nnwScdOmW^h=1b@O`O_u-&^E^NL2lmUh$L9H+2=JMH+Q>Y)PH;r1X>b?hs0AoI0 zs&0e#O(;;KZ+Zg$G28IH?_z=bynlp0$!_nT;15z8ptxIA2lPwx^ zK#i`33wiz()DAS-q}M22cP8(oV0)4FGWD3tbf3C8^hJS-zfZxKQZkl_Q6m`Nu~ub* zj_7AlIB|3NE#5@HT)b!C!qry24?>TBK|W9J2KxIL6i=HezA=JgMvY)xc%WB1Eg^3& z(xAE452<^@_jCibPB6aTVXXCNHNVzR?6pwqi|{5EYCZ7QO?-MGk~Gnab4_9NhN#)4 zch#|`x7TdFGs=1)`5K^?V2px|^hT@MrS~jL^!C7BeAPVcx*5ZAT}3Sn{sYt=R51E= zBejeg!6skDHYujgJ#(ECjQdv_3l^@1rsCIm!iLA%X{pZjMKSGFAI|U@TGbPS7ep}3 zs0~?X_##<`F%xK|ZyT0<4h&8kv0|8|FwsLnwMAHv9&i!_9w8HSU z#{t7kTfD*S?Fhz@-^g&Znq7udvJ7Kev@*Krl6?-pEiO}|3qS9vww7!6-OMO%m$Daq z6t_>o>?n>=BiJ;u>`mqKvrEPJiSyL$^QL!)$DV7nKq6uE!BRuV&HjzV((w>AyTtyLEV0%?u^HLt@I$eM5Q}A$PZ)P zNJnFjS~7q|G&=|1w{L8U9wTPhWcI-XlC z7?;Hw=?z!Ih{G~r{@cuC6>tG1}QJH{cuT3L(kkkblg9WtXvFupIL zti>)Dy@NSDg7Gs5 z#>N$`<~OeYLVJ~r|tNPnxBi`T#XfwFhKe|v+1 z+5U}DBN+1pzQeY{?~ts$`}{5fez^e?6G6P$%ul}g3udJJ5p{RinmjN;PcS=u9IfW} z4X?4`9&)y7oKihnjWdDfrxm&Sa$JJhl}O=g_&E)`(wRJ!I7sm_Dj5UA8Y|R9(q$PR>fL=fNj5Sog z%S>=Rf?!6C;MTy&cU^PHr~MCA2EC0C%;wX=)dFGhkFZ*#^Jzbmt$S(ephi@t1 zhs#YLOyBWt&#o|z;##vEUl(D=j2glAJGWwvC3@r6BOnk`%{V9BT1u;jBXS->2YBCIwu zOfWkd3Rep(MX&HArPmRU zn*T#|?+_UG1-Fjr$rq=m^=OEh$eZ8vB=_uk`@F_~!5KX|elF*WEpMt%qU=JSHo9cQjEHQHR(eF(aCKPlG@TT=~$Z%ws3ur<}ZwymjV zfZpWY-VOK%2sds`Re*!&SR3c=R_DULC?x~DS=*Yb5gEB~Vr@Y-SAspSim)%4=K%oQ zGaZCK$zCr{*9qL_N_CrE+HDBN*HtTXQ#%Z5uw~W;ZyNp~Y*??yTO1dL)z~ztQ4c-Y z(vhpHu*G?MB@CJ?A*m`Mp;ZFG=W&$~t~Lg0HY?$FkxKC10+&^LQ8j4oc1oopjH=sN zU1VnU&5Bi65E5uX1mimpM$?W~v&rf`#~Z6}SFHAHtP+evA|tEOYBpKD@0hX5xZztB ztCwo55_|>6YP6b7RzGsQvHA|PUnLkb+(uiCRkkyK z@GY}|XC;`81H#pC??Ic^y2=W#1C+hnlTVd3qh-iFsK{NWkxMY^U!&D*l6#gFatCP% zUxQrTCouG86upBQy#(V+;6_gyu7;oQvq|r;6?$QHS>xLX7tMCK47vLixv+#Q;1~#I z6NlkyINxrQ-1DrE8{7;phSSX}ZecUL0Jj9=&MQX44Ohb`*e198tZ>_NsF$O&Bg>TqZ;2S3$%^~QnAgSnPyD>V>sNvq2HhE1QLtdGfmS(~( z*4zNWmviUmq0Ks<~G`GxSDO3 z@F7WR{Z)5D>z@t@CP87L%>lT4YyHWpOvy=AoC{!K6QxAd`X+r{5Ht1kPGv3d*|tDK z{G@_m1(VrYNKGF!5scBjYKX1%O3Ru-!-ceNDW@kEdd!n!?^8wT(n|Po1$T2LBsD$Y zB^W1v^b*d86;-jE*Zu0&u+4XXGlFr9Vr*g2YQFRO0-C?2w8l$({)OUI?`uD(VEpWr(Oeldf^q6ZZ$QSm zS*DlzfV#UrCLkC$I4~9}TrDs+`$PFQ^iwO9%?`I1YTs|BmSEh&$4G6sT41Ifwrib2@T$(EL{%`?o}`w8jzYE8W0>v-CAIH zLBHVau5_sT8A0=y73!Ys9IoK!pxdYZ^dOQ6bI(>Lp@+FU70iaYq_#kd^a#dYO78>v zoWnE4QYB_)Y>T?TZixirw^WU;I$SN_s#lixpz$A9OTt%j9g%Rh;SxF@o6@;n8aTu*U+YJ0v-k_O>ep3qQ--E}qI2 zvgB$F*(b))E>=dOJK8f9%sN_9(;Y3rttBk2d_uQjxe{ZzOU%xIU|eHh^s?bkU1na()P3kIVQrbKuQ_;&5sjw?nAPKTw!MLN4kyKLCUn?OPH;~qQ)Ly4h zDa}e2F;%@>-Cpld2|kA_Rk&JUcJx*9RXt^;s=JZbc%FZSVpU&7Nidu6idOR*kOfvt zR#@$us^;^tr$J6p#OkRTg12(!8m<;dd08O#jaG>5D#NzbvC^|E&7=~{PJo1~1!6x7 zq~0M(D&{{dbXf;vpFy_2y+p9^vtliO+k_ZHn|@{9`jkJx!<^3uS8IFyLYCHdTcFk7 zDFOcBK1KT_bXShdK(YY#NqDjushy7@Yj^YYjPg!urROc4z>TP z!prL8bH4hz+8SoZ19f|UcLUfBqV;rPyFd&=8AlttKzstLfol5VqoX0wC=G4>(75i} z+Dqs7&~4XRS34{0>04(uXn+>=et0CR>m_BUbYm{#Y{?W0l~k?(8uSTHj2X2W?19bJ)YsH%OZM$o+`;GQh&sde82#bf zWAwAzV>}A%CRcmk!apz=G;WXa4LArT@*3y%t8-zl_1gv*CNpY%88?^DmYhQUK)o?M z{1Tj4>rI!6H^CFvy|2Je*c{`l@CP=>_!?a$aFZqKCNP!>+=O5peSRlI4aPDiHB7@j z25ODXF&=?`G~1h$RecxlahzCHV?^1gi0(`(o(EU2bNUVAdtmah+L$t-Iuf10pV2_XT}3 z#d2jD)`Y~O_GB})1mm9RMsGmJb!rL5^&nQg!3sOncF&ZGhapQA!x=0#^GYyI<{5dV z<2tVdTkpP3x5I1i4OORHiRlbhn3*M*bp~`?XO>{=&S0w@X1jseYP^8!Br~rBv)vaR z*Lfw_y2b7k<<+0llM36vF4_CSxcIXeH{#yajRA5XB?1$C&f%z`@j4h9CmTtFeMmGV z4KA4T(716^8VEP8xVE;l^MP>dD(hpym8KKd+JXPI| zlLy+;(KFWFb+x6lAFHmO?3>)ec6bZ>F*aPM~Sbsumabid}f6K|=% zrOs}dnNF3R#BLBYzYEo|<)T!+?9^-PhtfMz6WhnK>Fj8FbSgbElFnwvCq^blN4Jem zWU^y8%yxh8Ej?l0JnvK#=b#uzu?NK!C|-qP8buk!>rvc>;w}{LLGfV}pF{BoibqjA zj^also<;HBCg_k0jbP^?FBHi|J67ofNl#mi7QC~iRUCKR`$xCh03DDFq`X%t^V@f{RT zqj(0zZ&CDifmnuO9g1^MjHB3t;tCY6LNSe^jN)b#Z$ohxiua)S5Q@*Bcm&0xC>}@g z0~9|+@jDa)-5^$?I1|MXid`r!L2(s|SE86gaTvudDDFVE7o*sZ;?*cz6gQxF1B%;Gyc5O!C_V|oai4FP@5JmD zkSLcj3AI|VbR?m@@z8F@#rP@B`(I@7a4@@Qn&@F^V-PwxYl<6MK75T#W+1H|R~HD5JO;#hobr1jTz%{3VLd zqxd?C$5G(-SiGO2_#F!T_=0y53fx-V+lXQtirpx#0^xX%!Sgpb-yX%ezRRxfIL4gu z(QT=*VP|4uD(j34PmGU_XVatU@sXjC^w{=HgK^v?)_Wa_x1zWk#l0vVK=B}ozeDkD z6#suf9m z_P&b$`Ue!as@eN@6u&~z3;m{tE0Vl*D9%AKjsj=Syem-PJfN3FF@xguDBgzRT_7B9 z3d-By^c;rMM&{hlWcx&JPY(}ghPJ1-Iip!8Jv_dB+n6&pJTfve>^R%gW7~j8?;*HD z?V>`y2iEWuwYHU2aW5`MG z*s)_`Vmv*T%EDqwc;YMJ+Uki@Gc)BxHdRS&$rYwy6p!$O5!*!%vib10!b2rr;k^z2aQX1t;2;#rYn(f%&V?7RZa2VZYWZ+fyBW@_ zEzYHz<%h)nRBpQN-2yn|#or8nl7rr@bhW_EmZ+P-D$2mku2nFu@wu}DHM$zUrTKPH zQ&g+ssmX^xk$TNVTB`01OTGg|A{bu)F%~IYt%s(3=UOVI+%S^B$5z_SgQ<&U=DN64!MOb1SQm^M!B~a5c`VsDWT}kppI< zrb}-&7Zl&n3M*)cnoi_kgU7v18iOn{PGFs)OZPYg;~IEl;iA=4;h@DmWTC}9BGcl$ z_rrDDn$SRl;Qday44qOedQUUcOE4}@HPRccW|iK*w?pq>SuuNw15rT<(%tQii?QGJ ziePxHG~fjY#*N901c$3(y;vpqd!hv6ikXk$9Voja)$e^y^eHgO^#Kb{@d?qV@Vk|h zsMjN#kJsldSJtlgd8-u6_IZpN!6*CH?icQ;`#iezCaqagLD@bpT1_=ezo4Ca_}>tjCB}^F78YY|X-*^O>R^ z!?#kRSJ`N+2<{v+y#%xLMypw+cgPOCa|w~pRRrrHGQqpJ5II~8>%}U;yF>}b5P7kM zR(PuDQ`qY9mWn=w?@Ly1%%xjdp)HXA-BRwP^V_BjUyqjeDVU9x88w1&rH?Xvznn)M zEz_m9Y0)wjlHoew#-o9QK(jftbxtkU}q ztMqm-OnZc7`Yl}lA_hDI)yLVVRnFQw&Rd&yE!D= z&(TXE8Kbt_c1ZS3QF<{8aT(r0I2U-G=u_CrLR>ET6j&EF77KmI&g{XWGQ6jYpZCea zLKbamopFds!+W5sa@tDu>w0O7v$+#RBZAqrQ?8BR6q{DuTXpjbO~s z7+rO^8ouM^yXqfUXoWu!wO>Cw)t`0BnNn^B7g7d845r?%GP6tYUXI;xHS6sD+77!5 ziZf0LzK#)-+sn<|5{%#6Hd=1DTB}Wg&$3~;Ck{%^KKqv2H&e`x|>y}FldL`E?k@*i`Gdqtpso4Xbo4x4T-GMx>}T0Td~4ZqCO3u;*Azo zecUY?@uZ}K=RcGzzPsV_Y9s9Q-UYX5taocH-D|9(^&Up-TPXe!1uoI?aJ7;5JPMq@ z^l&MGcRGr#D0ZU2Aw^@h#d{6@%R_M^iZ`S9V-)W~@qQGaK=FALUq^x6iT7_Po<%Va z+L1SaVkHRQQ~Nudwfkvi@}pll6n$2X;EkL^30G@#D8gf>grq0I>UR#nYaQ4Qy)AGn K2VaV*ZvFoix$lAi literal 0 HcmV?d00001 diff --git a/.serena/memories/code_style_conventions.md b/.serena/memories/code_style_conventions.md new file mode 100644 index 0000000..932ff2f --- /dev/null +++ b/.serena/memories/code_style_conventions.md @@ -0,0 +1,58 @@ +# Code Style and Conventions + +## TypeScript Configuration +- **Strict mode enabled**: All strict checks are on +- **Target**: ES2022 +- **Module**: ESNext with bundler resolution +- **Path aliases**: `@stock-bot/*` maps to `libs/*/src` +- **Decorators**: Enabled for dependency injection + +## Code Style Rules (ESLint) +- **No unused variables**: Error (except prefixed with `_`) +- **No explicit any**: Warning +- **No non-null assertion**: Warning +- **No console**: Warning (except in tests) +- **Prefer const**: Enforced +- **Strict equality**: Always use `===` +- **Curly braces**: Required for all blocks + +## Formatting (Prettier) +- **Semicolons**: Always +- **Single quotes**: Yes +- **Trailing comma**: ES5 +- **Print width**: 100 characters +- **Tab width**: 2 spaces +- **Arrow parens**: Avoid when possible +- **End of line**: LF + +## Import Order +1. Node built-ins +2. Third-party modules +3. `@stock-bot/*` imports +4. Relative imports (parent directories first) +5. Current directory imports + +## Naming Conventions +- **Files**: kebab-case (e.g., `database-setup.ts`) +- **Classes**: PascalCase +- **Functions/Variables**: camelCase +- **Constants**: UPPER_SNAKE_CASE +- **Interfaces/Types**: PascalCase with 'I' or 'T' prefix optional + +## Library Standards +- **Named exports only**: No default exports +- **Factory patterns**: For complex initialization +- **Singleton pattern**: For global services (config, logger) +- **Direct class exports**: For DI-managed services + +## Testing +- **File naming**: `*.test.ts` or `*.spec.ts` +- **Test structure**: Bun's built-in test runner +- **Integration tests**: Use TestContainers for databases +- **Mocking**: Mock external dependencies + +## Documentation +- **JSDoc**: For all public APIs +- **README.md**: Required for each library +- **Usage examples**: Include in documentation +- **Error messages**: Descriptive with context \ No newline at end of file diff --git a/.serena/memories/current_refactoring_context.md b/.serena/memories/current_refactoring_context.md new file mode 100644 index 0000000..39bed86 --- /dev/null +++ b/.serena/memories/current_refactoring_context.md @@ -0,0 +1,41 @@ +# Current Refactoring Context + +## Data Ingestion Service Refactor +The project is currently undergoing a major refactoring to move away from singleton patterns to a dependency injection approach using service containers. + +### What's Been Done +- Created connection pool pattern with `ServiceContainer` +- Refactored data-ingestion service to use DI container +- Updated handlers to accept container parameter +- Added proper resource disposal with `ctx.dispose()` + +### Migration Status +- QM handler: ✅ Fully migrated to container pattern +- IB handler: ⚠️ Partially migrated (using migration helper) +- Proxy handler: ✅ Updated to accept container +- WebShare handler: ✅ Updated to accept container + +### Key Patterns +1. **Service Container**: Central DI container managing all connections +2. **Operation Context**: Provides scoped database access within operations +3. **Factory Pattern**: Connection factories for different databases +4. **Resource Disposal**: Always call `ctx.dispose()` after operations + +### Example Pattern +```typescript +const ctx = OperationContext.create('handler', 'operation', { container }); +try { + // Use databases through context + await ctx.mongodb.insertOne(data); + await ctx.postgres.query('...'); + return { success: true }; +} finally { + await ctx.dispose(); // Always cleanup +} +``` + +### Next Steps +- Complete migration of remaining IB operations +- Remove migration helper once complete +- Apply same pattern to other services +- Add monitoring for connection pools \ No newline at end of file diff --git a/.serena/memories/project_overview.md b/.serena/memories/project_overview.md new file mode 100644 index 0000000..3d6df9b --- /dev/null +++ b/.serena/memories/project_overview.md @@ -0,0 +1,55 @@ +# Stock Bot Trading Platform + +## Project Purpose +This is an advanced trading bot platform with a microservice architecture designed for automated stock trading. The system includes: +- Market data ingestion from multiple providers (Yahoo Finance, QuoteMedia, Interactive Brokers, WebShare) +- Data processing and technical indicator calculation +- Trading strategy development and backtesting +- Order execution and risk management +- Portfolio tracking and performance analytics +- Web dashboard for monitoring + +## Architecture Overview +The project follows a **microservices architecture** with shared libraries: + +### Core Services (apps/) +- **data-ingestion**: Ingests market data from multiple providers +- **data-pipeline**: Processes and transforms data +- **web-api**: REST API service +- **web-app**: React-based dashboard + +### Shared Libraries (libs/) +**Core Libraries:** +- config: Environment configuration with Zod validation +- logger: Structured logging with Loki integration +- di: Dependency injection container +- types: Shared TypeScript types +- handlers: Common handler patterns + +**Data Libraries:** +- postgres: PostgreSQL client for transactional data +- questdb: Time-series database for market data +- mongodb: Document storage for configurations + +**Service Libraries:** +- queue: BullMQ-based job processing +- event-bus: Dragonfly/Redis event bus +- shutdown: Graceful shutdown management + +**Utils:** +- Financial calculations and technical indicators +- Date utilities +- Position sizing calculations + +## Database Strategy +- **PostgreSQL**: Transactional data (orders, positions, strategies) +- **QuestDB**: Time-series data (OHLCV, indicators, performance metrics) +- **MongoDB**: Document storage (configurations, raw API responses) +- **Dragonfly/Redis**: Event bus and caching layer + +## Current Development Phase +Phase 1: Data Foundation Layer (In Progress) +- Enhancing data provider reliability +- Implementing data validation +- Optimizing time-series storage +- Building robust HTTP client with circuit breakers \ No newline at end of file diff --git a/.serena/memories/project_structure.md b/.serena/memories/project_structure.md new file mode 100644 index 0000000..63e2044 --- /dev/null +++ b/.serena/memories/project_structure.md @@ -0,0 +1,62 @@ +# Project Structure + +## Root Directory +``` +stock-bot/ +├── apps/ # Microservice applications +│ ├── data-ingestion/ # Market data ingestion service +│ ├── data-pipeline/ # Data processing pipeline +│ ├── web-api/ # REST API service +│ └── web-app/ # React dashboard +├── libs/ # Shared libraries +│ ├── core/ # Core functionality +│ │ ├── config/ # Configuration management +│ │ ├── logger/ # Logging infrastructure +│ │ ├── di/ # Dependency injection +│ │ ├── types/ # Shared TypeScript types +│ │ └── handlers/ # Common handler patterns +│ ├── data/ # Database clients +│ │ ├── postgres/ # PostgreSQL client +│ │ ├── questdb/ # QuestDB time-series client +│ │ └── mongodb/ # MongoDB document storage +│ ├── services/ # Service utilities +│ │ ├── queue/ # BullMQ job processing +│ │ ├── event-bus/ # Dragonfly event bus +│ │ └── shutdown/ # Graceful shutdown +│ └── utils/ # Utility functions +├── database/ # Database schemas and migrations +├── scripts/ # Build and utility scripts +├── config/ # Configuration files +├── monitoring/ # Monitoring configurations +├── docs/ # Documentation +└── test/ # Global test utilities + +## Key Files +- `package.json` - Root package configuration +- `turbo.json` - Turbo monorepo configuration +- `tsconfig.json` - TypeScript configuration +- `eslint.config.js` - ESLint rules +- `.prettierrc` - Prettier formatting rules +- `docker-compose.yml` - Infrastructure setup +- `.env` - Environment variables + +## Monorepo Structure +- Uses Bun workspaces with Turbo for orchestration +- Each app and library has its own package.json +- Shared dependencies at root level +- Libraries published as `@stock-bot/*` packages + +## Service Architecture Pattern +Each service typically follows: +``` +service/ +├── src/ +│ ├── index.ts # Entry point +│ ├── routes/ # API routes (Hono) +│ ├── handlers/ # Business logic +│ ├── services/ # Service layer +│ └── types/ # Service-specific types +├── test/ # Tests +├── package.json +└── tsconfig.json +``` \ No newline at end of file diff --git a/.serena/memories/suggested_commands.md b/.serena/memories/suggested_commands.md new file mode 100644 index 0000000..57d0915 --- /dev/null +++ b/.serena/memories/suggested_commands.md @@ -0,0 +1,73 @@ +# Suggested Commands for Development + +## Package Management (Bun) +- `bun install` - Install all dependencies +- `bun add ` - Add a new dependency +- `bun add -D ` - Add a dev dependency +- `bun update` - Update dependencies + +## Development +- `bun run dev` - Start all services in development mode (uses Turbo) +- `bun run dev:full` - Start infrastructure + admin tools + dev mode +- `bun run dev:clean` - Reset infrastructure and start fresh + +## Building +- `bun run build` - Build all services and libraries +- `bun run build:libs` - Build only shared libraries +- `bun run build:all:clean` - Clean build with cache removal +- `./scripts/build-all.sh` - Custom build script with options + +## Testing +- `bun test` - Run all tests +- `bun test --watch` - Run tests in watch mode +- `bun run test:coverage` - Run tests with coverage report +- `bun run test:libs` - Test only shared libraries +- `bun run test:apps` - Test only applications +- `bun test ` - Run specific test file + +## Code Quality (IMPORTANT - Run before committing!) +- `bun run lint` - Check for linting errors +- `bun run lint:fix` - Auto-fix linting issues +- `bun run format` - Format code with Prettier +- `./scripts/format.sh` - Alternative format script + +## Infrastructure Management +- `bun run infra:up` - Start databases (PostgreSQL, QuestDB, MongoDB, Dragonfly) +- `bun run infra:down` - Stop infrastructure +- `bun run infra:reset` - Reset with clean volumes +- `bun run docker:admin` - Start admin GUIs (pgAdmin, Mongo Express, Redis Insight) +- `bun run docker:monitoring` - Start monitoring stack + +## Database Operations +- `bun run db:setup-ib` - Setup Interactive Brokers database schema +- `bun run db:init` - Initialize all database schemas + +## Utility Commands +- `bun run clean` - Clean build artifacts +- `bun run clean:all` - Deep clean including node_modules +- `turbo run ` - Run task across monorepo + +## Git Commands (Linux) +- `git status` - Check current status +- `git add .` - Stage all changes +- `git commit -m "message"` - Commit changes +- `git push` - Push to remote +- `git pull` - Pull from remote +- `git checkout -b ` - Create new branch + +## System Commands (Linux) +- `ls -la` - List files with details +- `cd ` - Change directory +- `grep -r "pattern" .` - Search for pattern +- `find . -name "*.ts"` - Find files by pattern +- `which ` - Find command location + +## MCP Setup (for database access in IDE) +- `./scripts/setup-mcp.sh` - Setup Model Context Protocol servers +- Requires infrastructure to be running first + +## Service URLs +- Dashboard: http://localhost:4200 +- QuestDB Console: http://localhost:9000 +- Grafana: http://localhost:3000 +- pgAdmin: http://localhost:8080 \ No newline at end of file diff --git a/.serena/memories/task_completion_checklist.md b/.serena/memories/task_completion_checklist.md new file mode 100644 index 0000000..fb82656 --- /dev/null +++ b/.serena/memories/task_completion_checklist.md @@ -0,0 +1,55 @@ +# Task Completion Checklist + +When you complete any coding task, ALWAYS run these commands in order: + +## 1. Code Quality Checks (MANDATORY) +```bash +# Run linting to catch code issues +bun run lint + +# If there are errors, fix them automatically +bun run lint:fix + +# Format the code +bun run format +``` + +## 2. Testing (if applicable) +```bash +# Run tests if you modified existing functionality +bun test + +# Run specific test file if you added/modified tests +bun test +``` + +## 3. Build Verification (for significant changes) +```bash +# Build the affected libraries/apps +bun run build:libs # if you changed libraries +bun run build # for full build +``` + +## 4. Final Verification Steps +- Ensure no TypeScript errors in the IDE +- Check that imports are properly ordered (Prettier should handle this) +- Verify no console.log statements in production code +- Confirm all new code follows the established patterns + +## 5. Git Commit Guidelines +- Stage changes: `git add .` +- Write descriptive commit messages +- Reference issue numbers if applicable +- Use conventional commit format when possible: + - `feat:` for new features + - `fix:` for bug fixes + - `refactor:` for code refactoring + - `docs:` for documentation + - `test:` for tests + - `chore:` for maintenance + +## Important Notes +- NEVER skip the linting and formatting steps +- The project uses ESLint and Prettier - let them do their job +- If lint errors persist after auto-fix, they need manual attention +- Always test your changes, even if just running the service locally \ No newline at end of file diff --git a/.serena/memories/tech_stack.md b/.serena/memories/tech_stack.md new file mode 100644 index 0000000..9697803 --- /dev/null +++ b/.serena/memories/tech_stack.md @@ -0,0 +1,49 @@ +# Technology Stack + +## Runtime & Package Manager +- **Bun**: v1.1.0+ (primary runtime and package manager) +- **Node.js**: v18.0.0+ (compatibility) +- **TypeScript**: v5.8.3 + +## Core Technologies +- **Turbo**: Monorepo build system +- **ESBuild**: Fast bundling (integrated with Bun) +- **Hono**: Lightweight web framework for services + +## Databases +- **PostgreSQL**: Primary transactional database +- **QuestDB**: Time-series database for market data +- **MongoDB**: Document storage +- **Dragonfly**: Redis-compatible cache and event bus + +## Queue & Messaging +- **BullMQ**: Job queue processing +- **IORedis**: Redis client for Dragonfly + +## Web Technologies +- **React**: Frontend framework (web-app) +- **Angular**: (based on polyfills.ts reference) +- **PrimeNG**: UI component library +- **TailwindCSS**: CSS framework + +## Testing +- **Bun Test**: Built-in test runner +- **TestContainers**: Database integration testing +- **Supertest**: API testing + +## Monitoring & Observability +- **Loki**: Log aggregation +- **Prometheus**: Metrics collection +- **Grafana**: Visualization dashboards + +## Development Tools +- **ESLint**: Code linting +- **Prettier**: Code formatting +- **Docker Compose**: Local infrastructure +- **Model Context Protocol (MCP)**: Database access in IDE + +## Key Dependencies +- **Awilix**: Dependency injection container +- **Zod**: Schema validation +- **pg**: PostgreSQL client +- **Playwright**: Browser automation for proxy testing \ No newline at end of file diff --git a/.serena/project.yml b/.serena/project.yml new file mode 100644 index 0000000..68debf8 --- /dev/null +++ b/.serena/project.yml @@ -0,0 +1,66 @@ +# language of the project (csharp, python, rust, java, typescript, javascript, go, cpp, or ruby) +# Special requirements: +# * csharp: Requires the presence of a .sln file in the project folder. +language: typescript + +# whether to use the project's gitignore file to ignore files +# Added on 2025-04-07 +ignore_all_files_in_gitignore: true +# list of additional paths to ignore +# same syntax as gitignore, so you can use * and ** +# Was previously called `ignored_dirs`, please update your config if you are using that. +# Added (renamed)on 2025-04-07 +ignored_paths: [] + +# whether the project is in read-only mode +# If set to true, all editing tools will be disabled and attempts to use them will result in an error +# Added on 2025-04-18 +read_only: false + + +# list of tool names to exclude. We recommend not excluding any tools, see the readme for more details. +# Below is the complete list of tools for convenience. +# To make sure you have the latest list of tools, and to view their descriptions, +# execute `uv run scripts/print_tool_overview.py`. +# +# * `activate_project`: Activates a project by name. +# * `check_onboarding_performed`: Checks whether project onboarding was already performed. +# * `create_text_file`: Creates/overwrites a file in the project directory. +# * `delete_lines`: Deletes a range of lines within a file. +# * `delete_memory`: Deletes a memory from Serena's project-specific memory store. +# * `execute_shell_command`: Executes a shell command. +# * `find_referencing_code_snippets`: Finds code snippets in which the symbol at the given location is referenced. +# * `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type). +# * `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). +# * `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes. +# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file or directory. +# * `initial_instructions`: Gets the initial instructions for the current project. +# Should only be used in settings where the system prompt cannot be set, +# e.g. in clients you have no control over, like Claude Desktop. +# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol. +# * `insert_at_line`: Inserts content at a given line in a file. +# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol. +# * `list_dir`: Lists files and directories in the given directory (optionally with recursion). +# * `list_memories`: Lists memories in Serena's project-specific memory store. +# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building). +# * `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context). +# * `read_file`: Reads a file within the project directory. +# * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store. +# * `remove_project`: Removes a project from the Serena configuration. +# * `replace_lines`: Replaces a range of lines within a file with new content. +# * `replace_symbol_body`: Replaces the full definition of a symbol. +# * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen. +# * `search_for_pattern`: Performs a search for a pattern in the project. +# * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase. +# * `switch_modes`: Activates modes by providing a list of their names +# * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information. +# * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task. +# * `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed. +# * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store. +excluded_tools: [] + +# initial prompt for the project. It will always be given to the LLM upon activating the project +# (contrary to the memories, which are loaded on demand). +initial_prompt: "" + +project_name: "stock-bot" diff --git a/.vscode/mcp.json b/.vscode/mcp.json index c77541f..0e0dcd2 100644 --- a/.vscode/mcp.json +++ b/.vscode/mcp.json @@ -1,21 +1,3 @@ { - "mcpServers": { - "postgres": { - "command": "npx", - "args": [ - "-y", - "@modelcontextprotocol/server-postgres", - "postgresql://trading_user:trading_pass_dev@localhost:5432/trading_bot" - ] - }, - "mongodb": { - "command": "npx", - "args": [ - "-y", - "mongodb-mcp-server", - "--connectionString", - "mongodb://trading_admin:trading_mongo_dev@localhost:27017/stock?authSource=admin" - ] - } - } + } \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 448cb84..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,171 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Development Commands - -**Package Manager**: Bun (v1.1.0+) - -**Build & Development**: -- `bun install` - Install dependencies -- `bun run dev` - Start all services in development mode (uses Turbo) -- `bun run build` - Build all services and libraries -- `bun run build:libs` - Build only shared libraries -- `./scripts/build-all.sh` - Custom build script with options - -**Testing**: -- `bun test` - Run all tests -- `bun run test:libs` - Test only shared libraries -- `bun run test:apps` - Test only applications -- `bun run test:coverage` - Run tests with coverage - -**Code Quality**: -- `bun run lint` - Lint TypeScript files -- `bun run lint:fix` - Auto-fix linting issues -- `bun run format` - Format code using Prettier -- `./scripts/format.sh` - Format script - -**Infrastructure**: -- `bun run infra:up` - Start database infrastructure (PostgreSQL, QuestDB, MongoDB, Dragonfly) -- `bun run infra:down` - Stop infrastructure -- `bun run infra:reset` - Reset infrastructure with clean volumes -- `bun run docker:admin` - Start admin GUIs (pgAdmin, Mongo Express, Redis Insight) - -**Database Setup**: -- `bun run db:setup-ib` - Setup Interactive Brokers database schema -- `bun run db:init` - Initialize database schemas - -## Architecture Overview - -**Microservices Architecture** with shared libraries and multi-database storage: - -### Core Services (`apps/`) -- **data-ingestion** - Market data ingestion from multiple providers (Yahoo, QuoteMedia, IB) -- **processing-service** - Data cleaning, validation, and technical indicators -- **strategy-service** - Trading strategies and backtesting (multi-mode: live, event-driven, vectorized, hybrid) -- **execution-service** - Order management and risk controls -- **portfolio-service** - Position tracking and performance analytics -- **web-app** - React dashboard with real-time updates - -### Shared Libraries (`libs/`) -- **config** - Environment configuration with Zod validation -- **logger** - Loki-integrated structured logging (use `getLogger()` pattern) -- **http** - HTTP client with proxy support and rate limiting -- **cache** - Redis/Dragonfly caching layer -- **queue** - BullMQ-based job processing with batch support -- **postgres-client** - PostgreSQL operations with transactions -- **questdb-client** - Time-series data storage -- **mongodb-client** - Document storage operations -- **utils** - Financial calculations and technical indicators - -### Database Strategy -- **PostgreSQL** - Transactional data (orders, positions, strategies) -- **QuestDB** - Time-series data (OHLCV, indicators, performance metrics) -- **MongoDB** - Document storage (configurations, raw responses) -- **Dragonfly** - Event bus and caching (Redis-compatible) - -## Key Patterns & Conventions - -**Library Usage**: -- Import from shared libraries: `import { getLogger } from '@stock-bot/logger'` -- Use configuration: `import { databaseConfig } from '@stock-bot/config'` -- Logger pattern: `const logger = getLogger('service-name')` - -**Service Structure**: -- Each service has `src/index.ts` as entry point -- Routes in `src/routes/` using Hono framework -- Handlers/services in `src/handlers/` or `src/services/` -- Use dependency injection pattern - -**Data Processing**: -- Raw data → QuestDB via handlers -- Processed data → PostgreSQL via processing service -- Event-driven communication via Dragonfly -- Queue-based batch processing for large datasets - -**Multi-Mode Backtesting**: -- **Live Mode** - Real-time trading with brokers -- **Event-Driven** - Realistic simulation with market conditions -- **Vectorized** - Fast mathematical backtesting for optimization -- **Hybrid** - Validation by comparing vectorized vs event-driven results - -## Development Workflow - -1. **Start Infrastructure**: `bun run infra:up` -2. **Build Libraries**: `bun run build:libs` -3. **Start Development**: `bun run dev` -4. **Access UIs**: - - Dashboard: http://localhost:4200 - - QuestDB Console: http://localhost:9000 - - Grafana: http://localhost:3000 - - pgAdmin: http://localhost:8080 - -## Important Files & Locations - -**Configuration**: -- Environment variables in `.env` files -- Service configs in `libs/config/src/` -- Database init scripts in `database/postgres/init/` - -**Key Scripts**: -- `scripts/build-all.sh` - Production build with cleanup -- `scripts/docker.sh` - Docker management -- `scripts/format.sh` - Code formatting -- `scripts/setup-mcp.sh` - Setup Model Context Protocol servers for database access - -**Documentation**: -- `SIMPLIFIED-ARCHITECTURE.md` - Detailed architecture overview -- `DEVELOPMENT-ROADMAP.md` - Development phases and priorities -- Individual library READMEs in `libs/*/README.md` - -## Current Development Phase - -**Phase 1: Data Foundation Layer** (In Progress) -- Enhancing data provider reliability and rate limiting -- Implementing data validation and quality metrics -- Optimizing QuestDB storage for time-series data -- Building robust HTTP client with circuit breakers - -Focus on data quality and provider fault tolerance before advancing to strategy implementation. - -## Testing & Quality - -- Use Bun's built-in test runner -- Integration tests with TestContainers for databases -- ESLint for code quality with TypeScript rules -- Prettier for code formatting -- All services should have health check endpoints - -## Model Context Protocol (MCP) Setup - -**MCP Database Servers** are configured in `.vscode/mcp.json` for direct database access: - -- **PostgreSQL MCP Server**: Provides read-only access to PostgreSQL database - - Connection: `postgresql://trading_user:trading_pass_dev@localhost:5432/trading_bot` - - Package: `@modelcontextprotocol/server-postgres` - -- **MongoDB MCP Server**: Official MongoDB team server for database and Atlas interaction - - Connection: `mongodb://trading_admin:trading_mongo_dev@localhost:27017/stock?authSource=admin` - - Package: `mongodb-mcp-server` (official MongoDB JavaScript team package) - -**Setup Commands**: -- `./scripts/setup-mcp.sh` - Setup and test MCP servers -- `bun run infra:up` - Start database infrastructure (required for MCP) - -**Usage**: Once configured, Claude Code can directly query and inspect database schemas and data through natural language commands. - -## Environment Variables - -Key environment variables (see `.env` example): -- `NODE_ENV` - Environment (development/production) -- `DATA_SERVICE_PORT` - Port for data service -- `DRAGONFLY_HOST/PORT` - Cache/event bus connection -- Database connection strings for PostgreSQL, QuestDB, MongoDB - -## Monitoring & Observability - -- **Logging**: Structured JSON logs to Loki -- **Metrics**: Prometheus metrics collection -- **Visualization**: Grafana dashboards -- **Queue Monitoring**: Bull Board for job queues -- **Health Checks**: All services expose `/health` endpoints \ No newline at end of file diff --git a/MIGRATION-TO-CONNECTION-POOLS.md b/MIGRATION-TO-CONNECTION-POOLS.md deleted file mode 100644 index ebbef64..0000000 --- a/MIGRATION-TO-CONNECTION-POOLS.md +++ /dev/null @@ -1,183 +0,0 @@ -# Migration Guide: From Singleton to Connection Pool Pattern - -## Overview - -This guide explains how to migrate from the singleton anti-pattern to a proper connection pool pattern using the new `@stock-bot/connection-factory` library. - -## Current State (Singleton Anti-Pattern) - -```typescript -// ❌ Old pattern - global singleton -import { connectMongoDB, getMongoDBClient } from '@stock-bot/mongodb-client'; -import { connectPostgreSQL, getPostgreSQLClient } from '@stock-bot/postgres-client'; - -// Initialize once at startup -await connectMongoDB(config); -await connectPostgreSQL(config); - -// Use everywhere -const mongo = getMongoDBClient(); -const postgres = getPostgreSQLClient(); -``` - -### Problems with this approach: -- Global state makes testing difficult -- All operations share the same connection pool -- Can't optimize pool sizes for different use cases -- Memory leaks from persistent connections -- Hard to implement graceful shutdown - -## New Pattern (Connection Factory + Service Container) - -### Step 1: Set up Connection Factory - -```typescript -// ✅ New pattern - connection factory -import { setupServiceContainer } from './setup/database-setup'; - -// Initialize service container at startup -const container = await setupServiceContainer(); - -// Register cleanup -shutdown.register(async () => { - await container.dispose(); -}); -``` - -### Step 2: Update Handlers to Use Container - -```typescript -// ✅ Use OperationContext with container -export class MyHandler { - constructor(private readonly container: ServiceContainer) {} - - async handleOperation(data: any) { - const context = OperationContext.create('my-handler', 'operation', { - container: this.container - }); - - try { - // Connections are managed by the container - await context.mongodb.insertOne(data); - await context.postgres.query('...'); - await context.cache.set('key', 'value'); - } finally { - // Clean up resources - await context.dispose(); - } - } -} -``` - -### Step 3: Update Route Handlers - -```typescript -// Pass container to route handlers -export function createRoutes(container: ServiceContainer) { - const router = new Hono(); - const handler = new MyHandler(container); - - router.get('/data', async (c) => { - const result = await handler.handleOperation(c.req.query()); - return c.json(result); - }); - - return router; -} -``` - -## Migration Checklist - -### For Each Service: - -1. **Create database setup module** - ```typescript - // apps/[service-name]/src/setup/database-setup.ts - export async function setupServiceContainer(): Promise { - // Configure connection pools based on service needs - } - ``` - -2. **Update main index.ts** - - Remove direct `connectMongoDB()` and `connectPostgreSQL()` calls - - Replace with `setupServiceContainer()` - - Pass container to route handlers and job processors - -3. **Update handlers** - - Accept `ServiceContainer` in constructor - - Create `OperationContext` with container - - Remove direct database client imports - - Add `context.dispose()` in finally blocks - -4. **Update job handlers** - ```typescript - // Before - export async function myJobHandler(job: Job) { - const mongo = getMongoDBClient(); - // ... - } - - // After - export function createMyJobHandler(container: ServiceContainer) { - return async (job: Job) => { - const context = OperationContext.create('job', job.name, { - container - }); - try { - // Use context.mongodb, context.postgres, etc. - } finally { - await context.dispose(); - } - }; - } - ``` - -## Pool Size Recommendations - -The `PoolSizeCalculator` provides optimal pool sizes based on service type: - -| Service | Min | Max | Use Case | -|---------|-----|-----|----------| -| data-ingestion | 5 | 50 | High-volume batch imports | -| data-pipeline | 3 | 30 | Data processing pipelines | -| web-api | 2 | 10 | Low-latency API requests | -| processing-service | 2 | 20 | CPU-intensive operations | -| portfolio-service | 2 | 15 | Portfolio calculations | -| strategy-service | 3 | 25 | Strategy backtesting | - -## Benefits After Migration - -1. **Better Resource Management** - - Each service gets appropriately sized connection pools - - Automatic cleanup with dispose pattern - - No more connection leaks - -2. **Improved Testing** - - Easy to mock containers for tests - - No global state to reset between tests - - Can test with different configurations - -3. **Enhanced Performance** - - Optimized pool sizes per service - - Isolated pools for heavy operations - - Better connection reuse - -4. **Operational Benefits** - - Connection pool metrics per service - - Graceful shutdown handling - - Better error isolation - -## Backward Compatibility - -The `OperationContext` maintains backward compatibility: -- If no container is provided, it falls back to singleton pattern -- This allows gradual migration service by service -- Warning logs indicate when fallback is used - -## Example: Complete Service Migration - -See `/apps/data-ingestion/src/handlers/example-handler.ts` for a complete example of: -- Using the service container -- Creating operation contexts -- Handling batch operations with scoped containers -- Proper resource cleanup \ No newline at end of file diff --git a/apps/data-ingestion/config/default.json b/apps/data-ingestion/config/default.json index c8d21e3..a919adc 100644 --- a/apps/data-ingestion/config/default.json +++ b/apps/data-ingestion/config/default.json @@ -94,4 +94,4 @@ "burstSize": 20 } } -} \ No newline at end of file +} diff --git a/apps/data-ingestion/src/handlers/ceo/actions/index.ts b/apps/data-ingestion/src/handlers/ceo/actions/index.ts index c5ac377..247112b 100644 --- a/apps/data-ingestion/src/handlers/ceo/actions/index.ts +++ b/apps/data-ingestion/src/handlers/ceo/actions/index.ts @@ -1,3 +1,3 @@ -export { updateCeoChannels } from './update-ceo-channels.action'; -export { updateUniqueSymbols } from './update-unique-symbols.action'; -export { processIndividualSymbol } from './process-individual-symbol.action'; \ No newline at end of file +export { updateCeoChannels } from './update-ceo-channels.action'; +export { updateUniqueSymbols } from './update-unique-symbols.action'; +export { processIndividualSymbol } from './process-individual-symbol.action'; diff --git a/apps/data-ingestion/src/handlers/ceo/actions/process-individual-symbol.action.ts b/apps/data-ingestion/src/handlers/ceo/actions/process-individual-symbol.action.ts index b7a7daa..748abe2 100644 --- a/apps/data-ingestion/src/handlers/ceo/actions/process-individual-symbol.action.ts +++ b/apps/data-ingestion/src/handlers/ceo/actions/process-individual-symbol.action.ts @@ -1,111 +1,117 @@ -import { getRandomUserAgent } from '@stock-bot/utils'; -import type { CeoHandler } from '../ceo.handler'; - -export async function processIndividualSymbol(this: CeoHandler, payload: any, _context: any): Promise { - const { ceoId, symbol, timestamp } = payload; - const proxy = this.proxy?.getProxy(); - if(!proxy) { - this.logger.warn('No proxy available for processing individual CEO symbol'); - return; - } - - this.logger.debug('Processing individual CEO symbol', { - ceoId, - timestamp, - }); - try { - // Fetch detailed information for the individual symbol - const response = await this.http.get(`https://api.ceo.ca/api/get_spiels?channel=${ceoId}&load_more=top` - + (timestamp ? `&until=${timestamp}` : ''), - { - proxy: proxy, - headers: { - - 'User-Agent': getRandomUserAgent() - } - }); - - if (!response.ok) { - throw new Error(`Failed to fetch details for ceoId ${ceoId}: ${response.statusText}`); - } - - const data = await response.json(); - - const spielCount = data.spiels.length; - if(spielCount === 0) { - this.logger.warn(`No spiels found for ceoId ${ceoId}`); - return null; // No data to process - } - const latestSpielTime = data.spiels[0]?.timestamp; - const posts = data.spiels.map((spiel: any) => ({ - ceoId, - spiel: spiel.spiel, - spielReplyToId: spiel.spiel_reply_to_id, - spielReplyTo: spiel.spiel_reply_to, - spielReplyToName: spiel.spiel_reply_to_name, - spielReplyToEdited: spiel.spiel_reply_to_edited, - userId: spiel.user_id, - name: spiel.name, - timestamp: spiel.timestamp, - spielId: spiel.spiel_id, - color: spiel.color, - parentId: spiel.parent_id, - publicId: spiel.public_id, - parentChannel: spiel.parent_channel, - parentTimestamp: spiel.parent_timestamp, - votes: spiel.votes, - editable: spiel.editable, - edited: spiel.edited, - featured: spiel.featured, - verified: spiel.verified, - fake: spiel.fake, - bot: spiel.bot, - voted: spiel.voted, - flagged: spiel.flagged, - ownSpiel: spiel.own_spiel, - score: spiel.score, - savedId: spiel.saved_id, - savedTimestamp: spiel.saved_timestamp, - poll: spiel.poll, - votedInPoll: spiel.voted_in_poll - })); - - await this.mongodb.batchUpsert('ceoPosts', posts, ['spielId']); - this.logger.info(`Fetched ${spielCount} spiels for ceoId ${ceoId}`); - - // Update Shorts - const shortRes = await this.http.get(`https://api.ceo.ca/api/short_positions/one?symbol=${symbol}`, - { - proxy: proxy, - headers: { - 'User-Agent': getRandomUserAgent() - } - }); - - if (shortRes.ok) { - const shortData = await shortRes.json(); - if(shortData && shortData.positions) { - await this.mongodb.batchUpsert('ceoShorts', shortData.positions, ['id']); - } - - await this.scheduleOperation('process-individual-symbol', { - ceoId: ceoId, - timestamp: latestSpielTime - }); - } - - - - this.logger.info(`Successfully processed channel ${ceoId} and added channel ${ceoId} at timestamp ${latestSpielTime}`); - - return { ceoId, spielCount, timestamp }; - - } catch (error) { - this.logger.error('Failed to process individual symbol', { - error, - ceoId, - timestamp - }); - throw error; - } -} \ No newline at end of file +import { getRandomUserAgent } from '@stock-bot/utils'; +import type { CeoHandler } from '../ceo.handler'; + +export async function processIndividualSymbol( + this: CeoHandler, + payload: any, + _context: any +): Promise { + const { ceoId, symbol, timestamp } = payload; + const proxy = this.proxy?.getProxy(); + if (!proxy) { + this.logger.warn('No proxy available for processing individual CEO symbol'); + return; + } + + this.logger.debug('Processing individual CEO symbol', { + ceoId, + timestamp, + }); + try { + // Fetch detailed information for the individual symbol + const response = await this.http.get( + `https://api.ceo.ca/api/get_spiels?channel=${ceoId}&load_more=top` + + (timestamp ? `&until=${timestamp}` : ''), + { + proxy: proxy, + headers: { + 'User-Agent': getRandomUserAgent(), + }, + } + ); + + if (!response.ok) { + throw new Error(`Failed to fetch details for ceoId ${ceoId}: ${response.statusText}`); + } + + const data = await response.json(); + + const spielCount = data.spiels.length; + if (spielCount === 0) { + this.logger.warn(`No spiels found for ceoId ${ceoId}`); + return null; // No data to process + } + const latestSpielTime = data.spiels[0]?.timestamp; + const posts = data.spiels.map((spiel: any) => ({ + ceoId, + spiel: spiel.spiel, + spielReplyToId: spiel.spiel_reply_to_id, + spielReplyTo: spiel.spiel_reply_to, + spielReplyToName: spiel.spiel_reply_to_name, + spielReplyToEdited: spiel.spiel_reply_to_edited, + userId: spiel.user_id, + name: spiel.name, + timestamp: spiel.timestamp, + spielId: spiel.spiel_id, + color: spiel.color, + parentId: spiel.parent_id, + publicId: spiel.public_id, + parentChannel: spiel.parent_channel, + parentTimestamp: spiel.parent_timestamp, + votes: spiel.votes, + editable: spiel.editable, + edited: spiel.edited, + featured: spiel.featured, + verified: spiel.verified, + fake: spiel.fake, + bot: spiel.bot, + voted: spiel.voted, + flagged: spiel.flagged, + ownSpiel: spiel.own_spiel, + score: spiel.score, + savedId: spiel.saved_id, + savedTimestamp: spiel.saved_timestamp, + poll: spiel.poll, + votedInPoll: spiel.voted_in_poll, + })); + + await this.mongodb.batchUpsert('ceoPosts', posts, ['spielId']); + this.logger.info(`Fetched ${spielCount} spiels for ceoId ${ceoId}`); + + // Update Shorts + const shortRes = await this.http.get( + `https://api.ceo.ca/api/short_positions/one?symbol=${symbol}`, + { + proxy: proxy, + headers: { + 'User-Agent': getRandomUserAgent(), + }, + } + ); + + if (shortRes.ok) { + const shortData = await shortRes.json(); + if (shortData && shortData.positions) { + await this.mongodb.batchUpsert('ceoShorts', shortData.positions, ['id']); + } + + await this.scheduleOperation('process-individual-symbol', { + ceoId: ceoId, + timestamp: latestSpielTime, + }); + } + + this.logger.info( + `Successfully processed channel ${ceoId} and added channel ${ceoId} at timestamp ${latestSpielTime}` + ); + + return { ceoId, spielCount, timestamp }; + } catch (error) { + this.logger.error('Failed to process individual symbol', { + error, + ceoId, + timestamp, + }); + throw error; + } +} diff --git a/apps/data-ingestion/src/handlers/ceo/actions/update-ceo-channels.action.ts b/apps/data-ingestion/src/handlers/ceo/actions/update-ceo-channels.action.ts index 92c9bb5..d0be435 100644 --- a/apps/data-ingestion/src/handlers/ceo/actions/update-ceo-channels.action.ts +++ b/apps/data-ingestion/src/handlers/ceo/actions/update-ceo-channels.action.ts @@ -1,67 +1,72 @@ -import { getRandomUserAgent } from '@stock-bot/utils'; -import type { CeoHandler } from '../ceo.handler'; - -export async function updateCeoChannels(this: CeoHandler, payload: number | undefined): Promise { - const proxy = this.proxy?.getProxy(); - if(!proxy) { - this.logger.warn('No proxy available for CEO channels update'); - return; - } - let page; - if(payload === undefined) { - page = 1 - }else{ - page = payload; - } - - - this.logger.info(`Fetching CEO channels for page ${page} with proxy ${proxy}`); - const response = await this.http.get('https://api.ceo.ca/api/home?exchange=all&sort_by=symbol§or=All&tab=companies&page='+page, { - proxy: proxy, - headers: { - 'User-Agent': getRandomUserAgent() - } - }) - const results = await response.json(); - const channels = results.channel_categories[0].channels; - const totalChannels = results.channel_categories[0].total_channels; - const totalPages = Math.ceil(totalChannels / channels.length); - const exchanges: {exchange: string, countryCode: string}[] = [] - const symbols = channels.map((channel: any) =>{ - // check if exchange is in the exchanges array object - if(!exchanges.find((e: any) => e.exchange === channel.exchange)) { - exchanges.push({ - exchange: channel.exchange, - countryCode: 'CA' - }); - } - const details = channel.company_details || {}; - return { - symbol: channel.symbol, - exchange: channel.exchange, - name: channel.title, - type: channel.type, - ceoId: channel.channel, - marketCap: details.market_cap, - volumeRatio: details.volume_ratio, - avgVolume: details.avg_volume, - stockType: details.stock_type, - issueType: details.issue_type, - sharesOutstanding: details.shares_outstanding, - float: details.float, - } - }) - - await this.mongodb.batchUpsert('ceoSymbols', symbols, ['symbol', 'exchange']); - await this.mongodb.batchUpsert('ceoExchanges', exchanges, ['exchange']); - - if(page === 1) { - for( let i = 2; i <= totalPages; i++) { - this.logger.info(`Scheduling page ${i} of ${totalPages} for CEO channels`); - await this.scheduleOperation('update-ceo-channels', i) - } - } - - this.logger.info(`Fetched CEO channels for page ${page}/${totalPages}`); - return { page, totalPages }; -} \ No newline at end of file +import { getRandomUserAgent } from '@stock-bot/utils'; +import type { CeoHandler } from '../ceo.handler'; + +export async function updateCeoChannels( + this: CeoHandler, + payload: number | undefined +): Promise { + const proxy = this.proxy?.getProxy(); + if (!proxy) { + this.logger.warn('No proxy available for CEO channels update'); + return; + } + let page; + if (payload === undefined) { + page = 1; + } else { + page = payload; + } + + this.logger.info(`Fetching CEO channels for page ${page} with proxy ${proxy}`); + const response = await this.http.get( + 'https://api.ceo.ca/api/home?exchange=all&sort_by=symbol§or=All&tab=companies&page=' + page, + { + proxy: proxy, + headers: { + 'User-Agent': getRandomUserAgent(), + }, + } + ); + const results = await response.json(); + const channels = results.channel_categories[0].channels; + const totalChannels = results.channel_categories[0].total_channels; + const totalPages = Math.ceil(totalChannels / channels.length); + const exchanges: { exchange: string; countryCode: string }[] = []; + const symbols = channels.map((channel: any) => { + // check if exchange is in the exchanges array object + if (!exchanges.find((e: any) => e.exchange === channel.exchange)) { + exchanges.push({ + exchange: channel.exchange, + countryCode: 'CA', + }); + } + const details = channel.company_details || {}; + return { + symbol: channel.symbol, + exchange: channel.exchange, + name: channel.title, + type: channel.type, + ceoId: channel.channel, + marketCap: details.market_cap, + volumeRatio: details.volume_ratio, + avgVolume: details.avg_volume, + stockType: details.stock_type, + issueType: details.issue_type, + sharesOutstanding: details.shares_outstanding, + float: details.float, + }; + }); + + await this.mongodb.batchUpsert('ceoSymbols', symbols, ['symbol', 'exchange']); + await this.mongodb.batchUpsert('ceoExchanges', exchanges, ['exchange']); + + if (page === 1) { + for (let i = 2; i <= totalPages; i++) { + this.logger.info(`Scheduling page ${i} of ${totalPages} for CEO channels`); + await this.scheduleOperation('update-ceo-channels', i); + } + } + + this.logger.info(`Fetched CEO channels for page ${page}/${totalPages}`); + return { page, totalPages }; +} diff --git a/apps/data-ingestion/src/handlers/ceo/actions/update-unique-symbols.action.ts b/apps/data-ingestion/src/handlers/ceo/actions/update-unique-symbols.action.ts index 655dbe0..5f8bb59 100644 --- a/apps/data-ingestion/src/handlers/ceo/actions/update-unique-symbols.action.ts +++ b/apps/data-ingestion/src/handlers/ceo/actions/update-unique-symbols.action.ts @@ -1,63 +1,71 @@ -import type { CeoHandler } from '../ceo.handler'; - -export async function updateUniqueSymbols(this: CeoHandler, _payload: unknown, _context: any): Promise { - this.logger.info('Starting update to get unique CEO symbols by ceoId'); - - try { - // Get unique ceoId values from the ceoSymbols collection - const uniqueCeoIds = await this.mongodb.collection('ceoSymbols').distinct('ceoId'); - - this.logger.info(`Found ${uniqueCeoIds.length} unique CEO IDs`); - - // Get detailed records for each unique ceoId (latest/first record) - const uniqueSymbols = []; - for (const ceoId of uniqueCeoIds) { - const symbol = await this.mongodb.collection('ceoSymbols') - .findOne({ ceoId }, { sort: { _id: -1 } }); // Get latest record - - if (symbol) { - uniqueSymbols.push(symbol); - } - } - - this.logger.info(`Retrieved ${uniqueSymbols.length} unique symbol records`); - - // Schedule individual jobs for each unique symbol - let scheduledJobs = 0; - for (const symbol of uniqueSymbols) { - // Schedule a job to process this individual symbol - await this.scheduleOperation('process-individual-symbol', { - ceoId: symbol.ceoId, - symbol: symbol.symbol, - }); - scheduledJobs++; - - // Add small delay to avoid overwhelming the queue - if (scheduledJobs % 10 === 0) { - this.logger.debug(`Scheduled ${scheduledJobs} jobs so far`); - } - } - - this.logger.info(`Successfully scheduled ${scheduledJobs} individual symbol update jobs`); - - // Cache the results for monitoring - await this.cacheSet('unique-symbols-last-run', { - timestamp: new Date().toISOString(), - totalUniqueIds: uniqueCeoIds.length, - totalRecords: uniqueSymbols.length, - scheduledJobs - }, 1800); // Cache for 30 minutes - - return { - success: true, - uniqueCeoIds: uniqueCeoIds.length, - uniqueRecords: uniqueSymbols.length, - scheduledJobs, - timestamp: new Date().toISOString() - }; - - } catch (error) { - this.logger.error('Failed to update unique CEO symbols', { error }); - throw error; - } -} \ No newline at end of file +import type { CeoHandler } from '../ceo.handler'; + +export async function updateUniqueSymbols( + this: CeoHandler, + _payload: unknown, + _context: any +): Promise { + this.logger.info('Starting update to get unique CEO symbols by ceoId'); + + try { + // Get unique ceoId values from the ceoSymbols collection + const uniqueCeoIds = await this.mongodb.collection('ceoSymbols').distinct('ceoId'); + + this.logger.info(`Found ${uniqueCeoIds.length} unique CEO IDs`); + + // Get detailed records for each unique ceoId (latest/first record) + const uniqueSymbols = []; + for (const ceoId of uniqueCeoIds) { + const symbol = await this.mongodb + .collection('ceoSymbols') + .findOne({ ceoId }, { sort: { _id: -1 } }); // Get latest record + + if (symbol) { + uniqueSymbols.push(symbol); + } + } + + this.logger.info(`Retrieved ${uniqueSymbols.length} unique symbol records`); + + // Schedule individual jobs for each unique symbol + let scheduledJobs = 0; + for (const symbol of uniqueSymbols) { + // Schedule a job to process this individual symbol + await this.scheduleOperation('process-individual-symbol', { + ceoId: symbol.ceoId, + symbol: symbol.symbol, + }); + scheduledJobs++; + + // Add small delay to avoid overwhelming the queue + if (scheduledJobs % 10 === 0) { + this.logger.debug(`Scheduled ${scheduledJobs} jobs so far`); + } + } + + this.logger.info(`Successfully scheduled ${scheduledJobs} individual symbol update jobs`); + + // Cache the results for monitoring + await this.cacheSet( + 'unique-symbols-last-run', + { + timestamp: new Date().toISOString(), + totalUniqueIds: uniqueCeoIds.length, + totalRecords: uniqueSymbols.length, + scheduledJobs, + }, + 1800 + ); // Cache for 30 minutes + + return { + success: true, + uniqueCeoIds: uniqueCeoIds.length, + uniqueRecords: uniqueSymbols.length, + scheduledJobs, + timestamp: new Date().toISOString(), + }; + } catch (error) { + this.logger.error('Failed to update unique CEO symbols', { error }); + throw error; + } +} diff --git a/apps/data-ingestion/src/handlers/ceo/ceo.handler.ts b/apps/data-ingestion/src/handlers/ceo/ceo.handler.ts index 22c5a94..443f97e 100644 --- a/apps/data-ingestion/src/handlers/ceo/ceo.handler.ts +++ b/apps/data-ingestion/src/handlers/ceo/ceo.handler.ts @@ -3,13 +3,9 @@ import { Handler, Operation, ScheduledOperation, - type IServiceContainer + type IServiceContainer, } from '@stock-bot/handlers'; -import { - processIndividualSymbol, - updateCeoChannels, - updateUniqueSymbols -} from './actions'; +import { processIndividualSymbol, updateCeoChannels, updateUniqueSymbols } from './actions'; @Handler('ceo') // @Disabled() @@ -18,21 +14,21 @@ export class CeoHandler extends BaseHandler { super(services); // Handler name read from @Handler decorator } - @ScheduledOperation('update-ceo-channels', '0 */15 * * *', { - priority: 7, - immediately: false, - description: 'Get all CEO symbols and exchanges' + @ScheduledOperation('update-ceo-channels', '0 */15 * * *', { + priority: 7, + immediately: false, + description: 'Get all CEO symbols and exchanges', }) updateCeoChannels = updateCeoChannels; @Operation('update-unique-symbols') - @ScheduledOperation('process-unique-symbols', '0 0 1 * *', { - priority: 5, - immediately: false, - description: 'Process unique CEO symbols and schedule individual jobs' + @ScheduledOperation('process-unique-symbols', '0 0 1 * *', { + priority: 5, + immediately: false, + description: 'Process unique CEO symbols and schedule individual jobs', }) updateUniqueSymbols = updateUniqueSymbols; @Operation('process-individual-symbol') processIndividualSymbol = processIndividualSymbol; -} \ No newline at end of file +} diff --git a/apps/data-ingestion/src/handlers/example-handler.ts b/apps/data-ingestion/src/handlers/example-handler.ts index 58682a1..f2c55eb 100644 --- a/apps/data-ingestion/src/handlers/example-handler.ts +++ b/apps/data-ingestion/src/handlers/example-handler.ts @@ -1,114 +1,107 @@ -import { OperationContext } from '@stock-bot/di'; -import type { ServiceContainer } from '@stock-bot/di'; - -/** - * Example handler showing how to use the new connection pooling pattern - */ -export class ExampleHandler { - constructor(private readonly container: ServiceContainer) {} - - /** - * Example operation using the enhanced OperationContext - */ - async performOperation(data: any): Promise { - // Create operation context with container - const context = new OperationContext( - 'example-handler', - 'perform-operation', - this.container, - { data } - ); - - try { - // Log operation start - context.logger.info('Starting operation', { data }); - - // Use MongoDB through service resolution - const mongodb = context.resolve('mongodb'); - const result = await mongodb.collection('test').insertOne(data); - context.logger.debug('MongoDB insert complete', { insertedId: result.insertedId }); - - // Use PostgreSQL through service resolution - const postgres = context.resolve('postgres'); - await postgres.query( - 'INSERT INTO operations (id, status) VALUES ($1, $2)', - [result.insertedId, 'completed'] - ); - - // Use cache through service resolution - const cache = context.resolve('cache'); - await cache.set(`operation:${result.insertedId}`, { - status: 'completed', - timestamp: new Date() - }); - - context.logger.info('Operation completed successfully'); - } catch (error) { - context.logger.error('Operation failed', { error }); - throw error; - } - } - - /** - * Example of batch operation with isolated connection pool - */ - async performBatchOperation(items: any[]): Promise { - // Create a scoped container for this batch operation - const scopedContainer = this.container.createScope(); - - const context = new OperationContext( - 'example-handler', - 'batch-operation', - scopedContainer, - { itemCount: items.length } - ); - - try { - context.logger.info('Starting batch operation', { itemCount: items.length }); - - // Get services once for the batch - const mongodb = context.resolve('mongodb'); - const cache = context.resolve('cache'); - - // Process items in parallel - const promises = items.map(async (item, index) => { - const itemContext = new OperationContext( - 'example-handler', - `batch-item-${index}`, - scopedContainer, - { item } - ); - - try { - await mongodb.collection('batch').insertOne(item); - await cache.set(`batch:${item.id}`, item); - } catch (error) { - itemContext.logger.error('Batch item failed', { error, itemIndex: index }); - throw error; - } - }); - - await Promise.all(promises); - context.logger.info('Batch operation completed'); - - } finally { - // Clean up scoped resources - await scopedContainer.dispose(); - } - } -} - -/** - * Example of how to use in a job handler - */ -export async function createExampleJobHandler(container: ServiceContainer) { - return async (job: any) => { - const handler = new ExampleHandler(container); - - if (job.data.type === 'batch') { - await handler.performBatchOperation(job.data.items); - } else { - await handler.performOperation(job.data); - } - }; -} \ No newline at end of file +import { OperationContext } from '@stock-bot/di'; +import type { ServiceContainer } from '@stock-bot/di'; + +/** + * Example handler showing how to use the new connection pooling pattern + */ +export class ExampleHandler { + constructor(private readonly container: ServiceContainer) {} + + /** + * Example operation using the enhanced OperationContext + */ + async performOperation(data: any): Promise { + // Create operation context with container + const context = new OperationContext('example-handler', 'perform-operation', this.container, { + data, + }); + + try { + // Log operation start + context.logger.info('Starting operation', { data }); + + // Use MongoDB through service resolution + const mongodb = context.resolve('mongodb'); + const result = await mongodb.collection('test').insertOne(data); + context.logger.debug('MongoDB insert complete', { insertedId: result.insertedId }); + + // Use PostgreSQL through service resolution + const postgres = context.resolve('postgres'); + await postgres.query('INSERT INTO operations (id, status) VALUES ($1, $2)', [ + result.insertedId, + 'completed', + ]); + + // Use cache through service resolution + const cache = context.resolve('cache'); + await cache.set(`operation:${result.insertedId}`, { + status: 'completed', + timestamp: new Date(), + }); + + context.logger.info('Operation completed successfully'); + } catch (error) { + context.logger.error('Operation failed', { error }); + throw error; + } + } + + /** + * Example of batch operation with isolated connection pool + */ + async performBatchOperation(items: any[]): Promise { + // Create a scoped container for this batch operation + const scopedContainer = this.container.createScope(); + + const context = new OperationContext('example-handler', 'batch-operation', scopedContainer, { + itemCount: items.length, + }); + + try { + context.logger.info('Starting batch operation', { itemCount: items.length }); + + // Get services once for the batch + const mongodb = context.resolve('mongodb'); + const cache = context.resolve('cache'); + + // Process items in parallel + const promises = items.map(async (item, index) => { + const itemContext = new OperationContext( + 'example-handler', + `batch-item-${index}`, + scopedContainer, + { item } + ); + + try { + await mongodb.collection('batch').insertOne(item); + await cache.set(`batch:${item.id}`, item); + } catch (error) { + itemContext.logger.error('Batch item failed', { error, itemIndex: index }); + throw error; + } + }); + + await Promise.all(promises); + context.logger.info('Batch operation completed'); + } finally { + // Clean up scoped resources + await scopedContainer.dispose(); + } + } +} + +/** + * Example of how to use in a job handler + */ +export async function createExampleJobHandler(container: ServiceContainer) { + return async (job: any) => { + const handler = new ExampleHandler(container); + + if (job.data.type === 'batch') { + await handler.performBatchOperation(job.data.items); + } else { + await handler.performOperation(job.data); + } + }; +} diff --git a/apps/data-ingestion/src/handlers/example/example.handler.ts b/apps/data-ingestion/src/handlers/example/example.handler.ts index 1b2d70c..9c2fe52 100644 --- a/apps/data-ingestion/src/handlers/example/example.handler.ts +++ b/apps/data-ingestion/src/handlers/example/example.handler.ts @@ -1,94 +1,94 @@ -/** - * Example Handler - Demonstrates ergonomic handler patterns - * Shows inline operations, service helpers, and scheduled operations - */ - -import { - BaseHandler, - Handler, - Operation, - ScheduledOperation, - type ExecutionContext, - type IServiceContainer -} from '@stock-bot/handlers'; - -@Handler('example') -export class ExampleHandler extends BaseHandler { - constructor(services: IServiceContainer) { - super(services); - } - - /** - * Simple inline operation - no separate action file needed - */ - @Operation('get-stats') - async getStats(): Promise<{ total: number; active: number; cached: boolean }> { - // Use collection helper for cleaner MongoDB access - const total = await this.collection('items').countDocuments(); - const active = await this.collection('items').countDocuments({ status: 'active' }); - - // Use cache helpers with automatic prefixing - const cached = await this.cacheGet('last-total'); - await this.cacheSet('last-total', total, 300); // 5 minutes - - // Use log helper with automatic handler context - this.log('info', 'Stats retrieved', { total, active }); - - return { total, active, cached: cached !== null }; - } - - /** - * Scheduled operation using combined decorator - */ - @ScheduledOperation('cleanup-old-items', '0 2 * * *', { - priority: 5, - description: 'Clean up items older than 30 days' - }) - async cleanupOldItems(): Promise<{ deleted: number }> { - const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); - - const result = await this.collection('items').deleteMany({ - createdAt: { $lt: thirtyDaysAgo } - }); - - this.log('info', 'Cleanup completed', { deleted: result.deletedCount }); - - // Schedule a follow-up task - await this.scheduleIn('generate-report', { type: 'cleanup' }, 60); // 1 minute - - return { deleted: result.deletedCount }; - } - - /** - * Operation that uses proxy service - */ - @Operation('fetch-external-data') - async fetchExternalData(input: { url: string }): Promise<{ data: any }> { - const proxyUrl = this.proxy.getProxy(); - - if (!proxyUrl) { - throw new Error('No proxy available'); - } - - // Use HTTP client with proxy - const response = await this.http.get(input.url, { - proxy: proxyUrl, - timeout: 10000 - }); - - // Cache the result - await this.cacheSet(`external:${input.url}`, response.data, 3600); - - return { data: response.data }; - } - - /** - * Complex operation that still uses action file - */ - @Operation('process-batch') - async processBatch(input: any, context: ExecutionContext): Promise { - // For complex operations, still use action files - const { processBatch } = await import('./actions/batch.action'); - return processBatch(this, input); - } -} \ No newline at end of file +/** + * Example Handler - Demonstrates ergonomic handler patterns + * Shows inline operations, service helpers, and scheduled operations + */ + +import { + BaseHandler, + Handler, + Operation, + ScheduledOperation, + type ExecutionContext, + type IServiceContainer, +} from '@stock-bot/handlers'; + +@Handler('example') +export class ExampleHandler extends BaseHandler { + constructor(services: IServiceContainer) { + super(services); + } + + /** + * Simple inline operation - no separate action file needed + */ + @Operation('get-stats') + async getStats(): Promise<{ total: number; active: number; cached: boolean }> { + // Use collection helper for cleaner MongoDB access + const total = await this.collection('items').countDocuments(); + const active = await this.collection('items').countDocuments({ status: 'active' }); + + // Use cache helpers with automatic prefixing + const cached = await this.cacheGet('last-total'); + await this.cacheSet('last-total', total, 300); // 5 minutes + + // Use log helper with automatic handler context + this.log('info', 'Stats retrieved', { total, active }); + + return { total, active, cached: cached !== null }; + } + + /** + * Scheduled operation using combined decorator + */ + @ScheduledOperation('cleanup-old-items', '0 2 * * *', { + priority: 5, + description: 'Clean up items older than 30 days', + }) + async cleanupOldItems(): Promise<{ deleted: number }> { + const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); + + const result = await this.collection('items').deleteMany({ + createdAt: { $lt: thirtyDaysAgo }, + }); + + this.log('info', 'Cleanup completed', { deleted: result.deletedCount }); + + // Schedule a follow-up task + await this.scheduleIn('generate-report', { type: 'cleanup' }, 60); // 1 minute + + return { deleted: result.deletedCount }; + } + + /** + * Operation that uses proxy service + */ + @Operation('fetch-external-data') + async fetchExternalData(input: { url: string }): Promise<{ data: any }> { + const proxyUrl = this.proxy.getProxy(); + + if (!proxyUrl) { + throw new Error('No proxy available'); + } + + // Use HTTP client with proxy + const response = await this.http.get(input.url, { + proxy: proxyUrl, + timeout: 10000, + }); + + // Cache the result + await this.cacheSet(`external:${input.url}`, response.data, 3600); + + return { data: response.data }; + } + + /** + * Complex operation that still uses action file + */ + @Operation('process-batch') + async processBatch(input: any, context: ExecutionContext): Promise { + // For complex operations, still use action files + const { processBatch } = await import('./actions/batch.action'); + return processBatch(this, input); + } +} diff --git a/apps/data-ingestion/src/handlers/ib/actions/fetch-exchanges-and-symbols.action.ts b/apps/data-ingestion/src/handlers/ib/actions/fetch-exchanges-and-symbols.action.ts new file mode 100644 index 0000000..4f0c2a9 --- /dev/null +++ b/apps/data-ingestion/src/handlers/ib/actions/fetch-exchanges-and-symbols.action.ts @@ -0,0 +1,38 @@ +import type { IbHandler } from '../ib.handler'; + +export async function fetchExchangesAndSymbols(this: IbHandler): Promise { + this.logger.info('Starting IB exchanges and symbols fetch job'); + + try { + // Fetch session headers first + const sessionHeaders = await this.fetchSession(); + if (!sessionHeaders) { + this.logger.error('Failed to get session headers for IB job'); + return { success: false, error: 'No session headers' }; + } + + this.logger.info('Session headers obtained, fetching exchanges...'); + + // Fetch exchanges + const exchanges = await this.fetchExchanges(); + this.logger.info('Fetched exchanges from IB', { count: exchanges?.length || 0 }); + + // Fetch symbols + this.logger.info('Fetching symbols...'); + const symbols = await this.fetchSymbols(); + this.logger.info('Fetched symbols from IB', { count: symbols?.length || 0 }); + + return { + success: true, + exchangesCount: exchanges?.length || 0, + symbolsCount: symbols?.length || 0, + }; + } catch (error) { + this.logger.error('Failed to fetch IB exchanges and symbols', { error }); + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }; + } +} + diff --git a/apps/data-ingestion/src/handlers/ib/operations/exchanges.operations.ts b/apps/data-ingestion/src/handlers/ib/actions/fetch-exchanges.action.ts similarity index 55% rename from apps/data-ingestion/src/handlers/ib/operations/exchanges.operations.ts rename to apps/data-ingestion/src/handlers/ib/actions/fetch-exchanges.action.ts index 4b173a1..34a9cf7 100644 --- a/apps/data-ingestion/src/handlers/ib/operations/exchanges.operations.ts +++ b/apps/data-ingestion/src/handlers/ib/actions/fetch-exchanges.action.ts @@ -1,16 +1,15 @@ -/** - * IB Exchanges Operations - Fetching exchange data from IB API - */ -import { OperationContext } from '@stock-bot/di'; -import type { ServiceContainer } from '@stock-bot/di'; - +import type { IbHandler } from '../ib.handler'; import { IB_CONFIG } from '../shared/config'; -export async function fetchExchanges(sessionHeaders: Record, container: ServiceContainer): Promise { - const ctx = OperationContext.create('ib', 'exchanges', { container }); - +export async function fetchExchanges(this: IbHandler): Promise { try { - ctx.logger.info('🔍 Fetching exchanges with session headers...'); + // First get session headers + const sessionHeaders = await this.fetchSession(); + if (!sessionHeaders) { + throw new Error('Failed to get session headers'); + } + + this.logger.info('🔍 Fetching exchanges with session headers...'); // The URL for the exchange data API const exchangeUrl = IB_CONFIG.BASE_URL + IB_CONFIG.EXCHANGE_API; @@ -28,7 +27,7 @@ export async function fetchExchanges(sessionHeaders: Record, con 'X-Requested-With': 'XMLHttpRequest', }; - ctx.logger.info('📤 Making request to exchange API...', { + this.logger.info('📤 Making request to exchange API...', { url: exchangeUrl, headerCount: Object.keys(requestHeaders).length, }); @@ -41,7 +40,7 @@ export async function fetchExchanges(sessionHeaders: Record, con }); if (!response.ok) { - ctx.logger.error('❌ Exchange API request failed', { + this.logger.error('❌ Exchange API request failed', { status: response.status, statusText: response.statusText, }); @@ -50,19 +49,18 @@ export async function fetchExchanges(sessionHeaders: Record, con const data = await response.json(); const exchanges = data?.exchanges || []; - ctx.logger.info('✅ Exchange data fetched successfully'); + this.logger.info('✅ Exchange data fetched successfully'); - ctx.logger.info('Saving IB exchanges to MongoDB...'); - await ctx.mongodb.batchUpsert('ibExchanges', exchanges, ['id', 'country_code']); - ctx.logger.info('✅ Exchange IB data saved to MongoDB:', { + this.logger.info('Saving IB exchanges to MongoDB...'); + await this.mongodb.batchUpsert('ibExchanges', exchanges, ['id', 'country_code']); + this.logger.info('✅ Exchange IB data saved to MongoDB:', { count: exchanges.length, }); return exchanges; } catch (error) { - ctx.logger.error('❌ Failed to fetch exchanges', { error }); + this.logger.error('❌ Failed to fetch exchanges', { error }); return null; - } finally { - await ctx.dispose(); } -} \ No newline at end of file +} + diff --git a/apps/data-ingestion/src/handlers/ib/actions/fetch-session.action.ts b/apps/data-ingestion/src/handlers/ib/actions/fetch-session.action.ts new file mode 100644 index 0000000..59104d1 --- /dev/null +++ b/apps/data-ingestion/src/handlers/ib/actions/fetch-session.action.ts @@ -0,0 +1,83 @@ +import { Browser } from '@stock-bot/browser'; +import type { IbHandler } from '../ib.handler'; +import { IB_CONFIG } from '../shared/config'; + +export async function fetchSession(this: IbHandler): Promise | undefined> { + try { + await Browser.initialize({ + headless: true, + timeout: IB_CONFIG.BROWSER_TIMEOUT, + blockResources: false, + }); + this.logger.info('✅ Browser initialized'); + + const { page } = await Browser.createPageWithProxy( + IB_CONFIG.BASE_URL + IB_CONFIG.PRODUCTS_PAGE, + IB_CONFIG.DEFAULT_PROXY + ); + this.logger.info('✅ Page created with proxy'); + + const headersPromise = new Promise | undefined>(resolve => { + let resolved = false; + + page.onNetworkEvent(event => { + if (event.url.includes('/webrest/search/product-types/summary')) { + if (event.type === 'request') { + try { + resolve(event.headers); + } catch (e) { + resolve(undefined); + this.logger.debug('Raw Summary Response error', { error: (e as Error).message }); + } + } + } + }); + + // Timeout fallback + setTimeout(() => { + if (!resolved) { + resolved = true; + this.logger.warn('Timeout waiting for headers'); + resolve(undefined); + } + }, IB_CONFIG.HEADERS_TIMEOUT); + }); + + this.logger.info('⏳ Waiting for page load...'); + await page.waitForLoadState('domcontentloaded', { timeout: IB_CONFIG.PAGE_LOAD_TIMEOUT }); + this.logger.info('✅ Page loaded'); + + //Products tabs + this.logger.info('🔍 Looking for Products tab...'); + const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]'); + await productsTab.waitFor({ timeout: IB_CONFIG.ELEMENT_TIMEOUT }); + this.logger.info('✅ Found Products tab'); + this.logger.info('🖱️ Clicking Products tab...'); + await productsTab.click(); + this.logger.info('✅ Products tab clicked'); + + // New Products Checkbox + this.logger.info('🔍 Looking for "New Products Only" radio button...'); + const radioButton = page.locator('span.checkbox-text:has-text("New Products Only")'); + await radioButton.waitFor({ timeout: IB_CONFIG.ELEMENT_TIMEOUT }); + this.logger.info(`🎯 Found "New Products Only" radio button`); + await radioButton.first().click(); + this.logger.info('✅ "New Products Only" radio button clicked'); + + // Wait for and return headers immediately when captured + this.logger.info('⏳ Waiting for headers to be captured...'); + const headers = await headersPromise; + page.close(); + if (headers) { + this.logger.info('✅ Headers captured successfully'); + } else { + this.logger.warn('⚠️ No headers were captured'); + } + + return headers; + } catch (error) { + this.logger.error('Failed to fetch IB symbol summary', { error }); + return; + } +} + diff --git a/apps/data-ingestion/src/handlers/ib/operations/symbols.operations.ts b/apps/data-ingestion/src/handlers/ib/actions/fetch-symbols.action.ts similarity index 58% rename from apps/data-ingestion/src/handlers/ib/operations/symbols.operations.ts rename to apps/data-ingestion/src/handlers/ib/actions/fetch-symbols.action.ts index 64a5a44..40c53c5 100644 --- a/apps/data-ingestion/src/handlers/ib/operations/symbols.operations.ts +++ b/apps/data-ingestion/src/handlers/ib/actions/fetch-symbols.action.ts @@ -1,18 +1,16 @@ -/** - * IB Symbols Operations - Fetching symbol data from IB API - */ -import { OperationContext } from '@stock-bot/di'; -import type { ServiceContainer } from '@stock-bot/di'; - +import type { IbHandler } from '../ib.handler'; import { IB_CONFIG } from '../shared/config'; -// Fetch symbols from IB using the session headers -export async function fetchSymbols(sessionHeaders: Record, container: ServiceContainer): Promise { - const ctx = OperationContext.create('ib', 'symbols', { container }); - +export async function fetchSymbols(this: IbHandler): Promise { try { - ctx.logger.info('🔍 Fetching symbols with session headers...'); - + // First get session headers + const sessionHeaders = await this.fetchSession(); + if (!sessionHeaders) { + throw new Error('Failed to get session headers'); + } + + this.logger.info('🔍 Fetching symbols with session headers...'); + // Prepare headers - include all session headers plus any additional ones const requestHeaders = { ...sessionHeaders, @@ -39,18 +37,15 @@ export async function fetchSymbols(sessionHeaders: Record, conta }; // Get Summary - const summaryResponse = await fetch( - IB_CONFIG.BASE_URL + IB_CONFIG.SUMMARY_API, - { - method: 'POST', - headers: requestHeaders, - proxy: IB_CONFIG.DEFAULT_PROXY, - body: JSON.stringify(requestBody), - } - ); + const summaryResponse = await fetch(IB_CONFIG.BASE_URL + IB_CONFIG.SUMMARY_API, { + method: 'POST', + headers: requestHeaders, + proxy: IB_CONFIG.DEFAULT_PROXY, + body: JSON.stringify(requestBody), + }); if (!summaryResponse.ok) { - ctx.logger.error('❌ Summary API request failed', { + this.logger.error('❌ Summary API request failed', { status: summaryResponse.status, statusText: summaryResponse.statusText, }); @@ -58,36 +53,33 @@ export async function fetchSymbols(sessionHeaders: Record, conta } const summaryData = await summaryResponse.json(); - ctx.logger.info('✅ IB Summary data fetched successfully', { + this.logger.info('✅ IB Summary data fetched successfully', { totalCount: summaryData[0].totalCount, }); const symbols = []; requestBody.pageSize = IB_CONFIG.PAGE_SIZE; const pageCount = Math.ceil(summaryData[0].totalCount / IB_CONFIG.PAGE_SIZE) || 0; - ctx.logger.info('Fetching Symbols for IB', { pageCount }); - + this.logger.info('Fetching Symbols for IB', { pageCount }); + const symbolPromises = []; for (let page = 1; page <= pageCount; page++) { requestBody.pageNumber = page; // Fetch symbols for the current page - const symbolsResponse = fetch( - IB_CONFIG.BASE_URL + IB_CONFIG.PRODUCTS_API, - { - method: 'POST', - headers: requestHeaders, - proxy: IB_CONFIG.DEFAULT_PROXY, - body: JSON.stringify(requestBody), - } - ); + const symbolsResponse = fetch(IB_CONFIG.BASE_URL + IB_CONFIG.PRODUCTS_API, { + method: 'POST', + headers: requestHeaders, + proxy: IB_CONFIG.DEFAULT_PROXY, + body: JSON.stringify(requestBody), + }); symbolPromises.push(symbolsResponse); } - + const responses = await Promise.all(symbolPromises); for (const response of responses) { if (!response.ok) { - ctx.logger.error('❌ Symbols API request failed', { + this.logger.error('❌ Symbols API request failed', { status: response.status, statusText: response.statusText, }); @@ -98,29 +90,28 @@ export async function fetchSymbols(sessionHeaders: Record, conta if (symJson && symJson.length > 0) { symbols.push(...symJson); } else { - ctx.logger.warn('⚠️ No symbols found in response'); + this.logger.warn('⚠️ No symbols found in response'); continue; } } - + if (symbols.length === 0) { - ctx.logger.warn('⚠️ No symbols fetched from IB'); + this.logger.warn('⚠️ No symbols fetched from IB'); return null; } - ctx.logger.info('✅ IB symbols fetched successfully, saving to DB...', { + this.logger.info('✅ IB symbols fetched successfully, saving to DB...', { totalSymbols: symbols.length, }); - await ctx.mongodb.batchUpsert('ib_symbols', symbols, ['symbol', 'exchangeId']); - ctx.logger.info('Saved IB symbols to DB', { + await this.mongodb.batchUpsert('ib_symbols', symbols, ['symbol', 'exchangeId']); + this.logger.info('Saved IB symbols to DB', { totalSymbols: symbols.length, }); return symbols; } catch (error) { - ctx.logger.error('❌ Failed to fetch symbols', { error }); + this.logger.error('❌ Failed to fetch symbols', { error }); return null; - } finally { - await ctx.dispose(); } -} \ No newline at end of file +} + diff --git a/apps/data-ingestion/src/handlers/ib/actions/index.ts b/apps/data-ingestion/src/handlers/ib/actions/index.ts new file mode 100644 index 0000000..04dde8e --- /dev/null +++ b/apps/data-ingestion/src/handlers/ib/actions/index.ts @@ -0,0 +1,5 @@ +export { fetchSession } from './fetch-session.action'; +export { fetchExchanges } from './fetch-exchanges.action'; +export { fetchSymbols } from './fetch-symbols.action'; +export { fetchExchangesAndSymbols } from './fetch-exchanges-and-symbols.action'; + diff --git a/apps/data-ingestion/src/handlers/ib/ib.handler.ts b/apps/data-ingestion/src/handlers/ib/ib.handler.ts index bc21ac8..4dabb1a 100644 --- a/apps/data-ingestion/src/handlers/ib/ib.handler.ts +++ b/apps/data-ingestion/src/handlers/ib/ib.handler.ts @@ -1,90 +1,33 @@ -/** - * Interactive Brokers Provider for new queue system - */ -import { getLogger } from '@stock-bot/logger'; import { - createJobHandler, - handlerRegistry, - type HandlerConfigWithSchedule, -} from '@stock-bot/queue'; -import type { ServiceContainer } from '@stock-bot/di'; + BaseHandler, + Handler, + Operation, + ScheduledOperation, + type IServiceContainer, +} from '@stock-bot/handlers'; +import { fetchExchanges, fetchExchangesAndSymbols, fetchSession, fetchSymbols } from './actions'; -const logger = getLogger('ib-provider'); +@Handler('ib') +export class IbHandler extends BaseHandler { + constructor(services: IServiceContainer) { + super(services); + } -// Initialize and register the IB provider -export function initializeIBProvider(container: ServiceContainer) { - logger.debug('Registering IB provider with scheduled jobs...'); + @Operation('fetch-session') + fetchSession = fetchSession; - const ibProviderConfig: HandlerConfigWithSchedule = { - name: 'ib', - operations: { - 'fetch-session': createJobHandler(async () => { - // payload contains session configuration (not used in current implementation) - logger.debug('Processing session fetch request'); - const { fetchSession } = await import('./operations/session.operations'); - return fetchSession(container); - }), + @Operation('fetch-exchanges') + fetchExchanges = fetchExchanges; - 'fetch-exchanges': createJobHandler(async () => { - // payload should contain session headers - logger.debug('Processing exchanges fetch request'); - const { fetchSession } = await import('./operations/session.operations'); - const { fetchExchanges } = await import('./operations/exchanges.operations'); - const sessionHeaders = await fetchSession(container); - if (sessionHeaders) { - return fetchExchanges(sessionHeaders, container); - } - throw new Error('Failed to get session headers'); - }), + @Operation('fetch-symbols') + fetchSymbols = fetchSymbols; - 'fetch-symbols': createJobHandler(async () => { - // payload should contain session headers - logger.debug('Processing symbols fetch request'); - const { fetchSession } = await import('./operations/session.operations'); - const { fetchSymbols } = await import('./operations/symbols.operations'); - const sessionHeaders = await fetchSession(container); - if (sessionHeaders) { - return fetchSymbols(sessionHeaders, container); - } - throw new Error('Failed to get session headers'); - }), - - 'ib-exchanges-and-symbols': createJobHandler(async () => { - // Legacy operation for scheduled jobs - logger.info('Fetching symbol summary from IB'); - const { fetchSession } = await import('./operations/session.operations'); - const { fetchExchanges } = await import('./operations/exchanges.operations'); - const { fetchSymbols } = await import('./operations/symbols.operations'); - - const sessionHeaders = await fetchSession(container); - logger.info('Fetched symbol summary from IB'); - - if (sessionHeaders) { - logger.debug('Fetching exchanges from IB'); - const exchanges = await fetchExchanges(sessionHeaders, container); - logger.info('Fetched exchanges from IB', { count: exchanges?.length }); - - logger.debug('Fetching symbols from IB'); - const symbols = await fetchSymbols(sessionHeaders, container); - logger.info('Fetched symbols from IB', { symbols }); - - return { exchangesCount: exchanges?.length, symbolsCount: symbols?.length }; - } - return null; - }), - }, - scheduledJobs: [ - { - type: 'ib-exchanges-and-symbols', - operation: 'ib-exchanges-and-symbols', - cronPattern: '0 0 * * 0', // Every Sunday at midnight - priority: 5, - description: 'Fetch and update IB exchanges and symbols data', - // immediately: true, // Don't run immediately during startup to avoid conflicts - }, - ], - }; - - handlerRegistry.registerWithSchedule(ibProviderConfig); - logger.debug('IB provider registered successfully with scheduled jobs'); + @Operation('ib-exchanges-and-symbols') + @ScheduledOperation('ib-exchanges-and-symbols', '0 0 * * 0', { + priority: 5, + description: 'Fetch and update IB exchanges and symbols data', + immediately: false, + }) + fetchExchangesAndSymbols = fetchExchangesAndSymbols; } + diff --git a/apps/data-ingestion/src/handlers/ib/operations/session.operations.ts b/apps/data-ingestion/src/handlers/ib/operations/session.operations.ts deleted file mode 100644 index 1898c4a..0000000 --- a/apps/data-ingestion/src/handlers/ib/operations/session.operations.ts +++ /dev/null @@ -1,91 +0,0 @@ -/** - * IB Session Operations - Browser automation for session headers - */ -import { Browser } from '@stock-bot/browser'; -import { OperationContext } from '@stock-bot/di'; -import type { ServiceContainer } from '@stock-bot/di'; - -import { IB_CONFIG } from '../shared/config'; - -export async function fetchSession(container: ServiceContainer): Promise | undefined> { - const ctx = OperationContext.create('ib', 'session', { container }); - - try { - await Browser.initialize({ - headless: true, - timeout: IB_CONFIG.BROWSER_TIMEOUT, - blockResources: false - }); - ctx.logger.info('✅ Browser initialized'); - - const { page } = await Browser.createPageWithProxy( - IB_CONFIG.BASE_URL + IB_CONFIG.PRODUCTS_PAGE, - IB_CONFIG.DEFAULT_PROXY - ); - ctx.logger.info('✅ Page created with proxy'); - - const headersPromise = new Promise | undefined>(resolve => { - let resolved = false; - - page.onNetworkEvent(event => { - if (event.url.includes('/webrest/search/product-types/summary')) { - if (event.type === 'request') { - try { - resolve(event.headers); - } catch (e) { - resolve(undefined); - ctx.logger.debug('Raw Summary Response error', { error: (e as Error).message }); - } - } - } - }); - - // Timeout fallback - setTimeout(() => { - if (!resolved) { - resolved = true; - ctx.logger.warn('Timeout waiting for headers'); - resolve(undefined); - } - }, IB_CONFIG.HEADERS_TIMEOUT); - }); - - ctx.logger.info('⏳ Waiting for page load...'); - await page.waitForLoadState('domcontentloaded', { timeout: IB_CONFIG.PAGE_LOAD_TIMEOUT }); - ctx.logger.info('✅ Page loaded'); - - //Products tabs - ctx.logger.info('🔍 Looking for Products tab...'); - const productsTab = page.locator('#productSearchTab[role=\"tab\"][href=\"#products\"]'); - await productsTab.waitFor({ timeout: IB_CONFIG.ELEMENT_TIMEOUT }); - ctx.logger.info('✅ Found Products tab'); - ctx.logger.info('🖱️ Clicking Products tab...'); - await productsTab.click(); - ctx.logger.info('✅ Products tab clicked'); - - // New Products Checkbox - ctx.logger.info('🔍 Looking for \"New Products Only\" radio button...'); - const radioButton = page.locator('span.checkbox-text:has-text(\"New Products Only\")'); - await radioButton.waitFor({ timeout: IB_CONFIG.ELEMENT_TIMEOUT }); - ctx.logger.info(`🎯 Found \"New Products Only\" radio button`); - await radioButton.first().click(); - ctx.logger.info('✅ \"New Products Only\" radio button clicked'); - - // Wait for and return headers immediately when captured - ctx.logger.info('⏳ Waiting for headers to be captured...'); - const headers = await headersPromise; - page.close(); - if (headers) { - ctx.logger.info('✅ Headers captured successfully'); - } else { - ctx.logger.warn('⚠️ No headers were captured'); - } - - return headers; - } catch (error) { - ctx.logger.error('Failed to fetch IB symbol summary', { error }); - return; - } finally { - await ctx.dispose(); - } -} \ No newline at end of file diff --git a/apps/data-ingestion/src/handlers/ib/shared/config.ts b/apps/data-ingestion/src/handlers/ib/shared/config.ts index 1f09326..91bf09c 100644 --- a/apps/data-ingestion/src/handlers/ib/shared/config.ts +++ b/apps/data-ingestion/src/handlers/ib/shared/config.ts @@ -8,16 +8,17 @@ export const IB_CONFIG = { EXCHANGE_API: '/webrest/exchanges', SUMMARY_API: '/webrest/search/product-types/summary', PRODUCTS_API: '/webrest/search/products-by-filters', - + // Browser configuration BROWSER_TIMEOUT: 10000, PAGE_LOAD_TIMEOUT: 20000, ELEMENT_TIMEOUT: 5000, HEADERS_TIMEOUT: 30000, - + // API configuration DEFAULT_PROXY: 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80', PAGE_SIZE: 500, PRODUCT_COUNTRIES: ['CA', 'US'], PRODUCT_TYPES: ['STK'], -}; \ No newline at end of file +}; + diff --git a/apps/data-ingestion/src/handlers/index.ts b/apps/data-ingestion/src/handlers/index.ts index 01c94cc..d1623f6 100644 --- a/apps/data-ingestion/src/handlers/index.ts +++ b/apps/data-ingestion/src/handlers/index.ts @@ -6,11 +6,12 @@ import type { IServiceContainer } from '@stock-bot/handlers'; import { autoRegisterHandlers } from '@stock-bot/handlers'; import { getLogger } from '@stock-bot/logger'; - // Import handlers for bundling (ensures they're included in the build) import './qm/qm.handler'; import './webshare/webshare.handler'; import './ceo/ceo.handler'; +import './ib/ib.handler'; + // Add more handler imports as needed const logger = getLogger('handler-init'); @@ -21,21 +22,17 @@ const logger = getLogger('handler-init'); export async function initializeAllHandlers(serviceContainer: IServiceContainer): Promise { try { // Auto-register all handlers in this directory - const result = await autoRegisterHandlers( - __dirname, - serviceContainer, - { - pattern: '.handler.', - exclude: ['test', 'spec'], - dryRun: false - } - ); - + const result = await autoRegisterHandlers(__dirname, serviceContainer, { + pattern: '.handler.', + exclude: ['test', 'spec'], + dryRun: false, + }); + logger.info('Handler auto-registration complete', { registered: result.registered, - failed: result.failed + failed: result.failed, }); - + if (result.failed.length > 0) { logger.error('Some handlers failed to register', { failed: result.failed }); } @@ -51,21 +48,20 @@ export async function initializeAllHandlers(serviceContainer: IServiceContainer) */ async function manualHandlerRegistration(serviceContainer: any): Promise { logger.warn('Falling back to manual handler registration'); - + try { // // Import and register handlers manually // const { QMHandler } = await import('./qm/qm.handler'); // const qmHandler = new QMHandler(serviceContainer); // qmHandler.register(); - + // const { WebShareHandler } = await import('./webshare/webshare.handler'); // const webShareHandler = new WebShareHandler(serviceContainer); // webShareHandler.register(); - - + logger.info('Manual handler registration complete'); } catch (error) { logger.error('Manual handler registration failed', { error }); throw error; } -} \ No newline at end of file +} diff --git a/apps/data-ingestion/src/handlers/proxy/operations/check.operations.ts b/apps/data-ingestion/src/handlers/proxy/operations/check.operations.ts index 390d823..3858737 100644 --- a/apps/data-ingestion/src/handlers/proxy/operations/check.operations.ts +++ b/apps/data-ingestion/src/handlers/proxy/operations/check.operations.ts @@ -1,24 +1,23 @@ /** * Proxy Check Operations - Checking proxy functionality */ -import type { ProxyInfo } from '@stock-bot/proxy'; import { OperationContext } from '@stock-bot/di'; import { getLogger } from '@stock-bot/logger'; +import type { ProxyInfo } from '@stock-bot/proxy'; import { fetch } from '@stock-bot/utils'; - import { PROXY_CONFIG } from '../shared/config'; /** * Check if a proxy is working */ export async function checkProxy(proxy: ProxyInfo): Promise { - const ctx = { - logger: getLogger('proxy-check'), + const ctx = { + logger: getLogger('proxy-check'), resolve: (_name: string) => { throw new Error(`Service container not available for proxy operations`); - } + }, } as any; - + let success = false; ctx.logger.debug(`Checking Proxy:`, { protocol: proxy.protocol, @@ -28,16 +27,17 @@ export async function checkProxy(proxy: ProxyInfo): Promise { try { // Test the proxy using fetch with proxy support - const proxyUrl = proxy.username && proxy.password - ? `${proxy.protocol}://${encodeURIComponent(proxy.username)}:${encodeURIComponent(proxy.password)}@${proxy.host}:${proxy.port}` - : `${proxy.protocol}://${proxy.host}:${proxy.port}`; - + const proxyUrl = + proxy.username && proxy.password + ? `${proxy.protocol}://${encodeURIComponent(proxy.username)}:${encodeURIComponent(proxy.password)}@${proxy.host}:${proxy.port}` + : `${proxy.protocol}://${proxy.host}:${proxy.port}`; + const response = await fetch(PROXY_CONFIG.CHECK_URL, { proxy: proxyUrl, signal: AbortSignal.timeout(PROXY_CONFIG.CHECK_TIMEOUT), - logger: ctx.logger + logger: ctx.logger, } as any); - + const data = await response.text(); const isWorking = response.ok; @@ -94,7 +94,11 @@ export async function checkProxy(proxy: ProxyInfo): Promise { /** * Update proxy data in cache with working/total stats and average response time */ -async function updateProxyInCache(proxy: ProxyInfo, isWorking: boolean, ctx: OperationContext): Promise { +async function updateProxyInCache( + proxy: ProxyInfo, + isWorking: boolean, + ctx: OperationContext +): Promise { const _cacheKey = `${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`; try { @@ -167,6 +171,6 @@ async function updateProxyInCache(proxy: ProxyInfo, isWorking: boolean, ctx: Ope function updateProxyStats(sourceId: string, success: boolean, ctx: OperationContext) { // Stats are now handled by the global ProxyManager ctx.logger.debug('Proxy check result', { sourceId, success }); - + // TODO: Integrate with global ProxyManager stats if needed -} \ No newline at end of file +} diff --git a/apps/data-ingestion/src/handlers/proxy/operations/query.operations.ts b/apps/data-ingestion/src/handlers/proxy/operations/query.operations.ts index 221e1ba..5778af3 100644 --- a/apps/data-ingestion/src/handlers/proxy/operations/query.operations.ts +++ b/apps/data-ingestion/src/handlers/proxy/operations/query.operations.ts @@ -1,9 +1,8 @@ /** * Proxy Query Operations - Getting active proxies from cache */ -import type { ProxyInfo } from '@stock-bot/proxy'; import { OperationContext } from '@stock-bot/di'; - +import type { ProxyInfo } from '@stock-bot/proxy'; import { PROXY_CONFIG } from '../shared/config'; /** @@ -17,7 +16,7 @@ export async function getRandomActiveProxy( minSuccessRate: number = 50 ): Promise { const ctx = OperationContext.create('proxy', 'get-random'); - + try { // Get all active proxy keys from cache const pattern = protocol @@ -56,7 +55,10 @@ export async function getRandomActiveProxy( return proxyData; } } catch (error) { - ctx.logger.debug('Error reading proxy from cache', { key, error: (error as Error).message }); + ctx.logger.debug('Error reading proxy from cache', { + key, + error: (error as Error).message, + }); continue; } } @@ -76,4 +78,4 @@ export async function getRandomActiveProxy( }); return null; } -} \ No newline at end of file +} diff --git a/apps/data-ingestion/src/handlers/proxy/operations/queue.operations.ts b/apps/data-ingestion/src/handlers/proxy/operations/queue.operations.ts index b46030f..4b34072 100644 --- a/apps/data-ingestion/src/handlers/proxy/operations/queue.operations.ts +++ b/apps/data-ingestion/src/handlers/proxy/operations/queue.operations.ts @@ -1,13 +1,13 @@ /** * Proxy Queue Operations - Queueing proxy operations */ +import { OperationContext } from '@stock-bot/di'; import type { ProxyInfo } from '@stock-bot/proxy'; import { QueueManager } from '@stock-bot/queue'; -import { OperationContext } from '@stock-bot/di'; export async function queueProxyFetch(): Promise { const ctx = OperationContext.create('proxy', 'queue-fetch'); - + const queueManager = QueueManager.getInstance(); const queue = queueManager.getQueue('proxy'); const job = await queue.add('proxy-fetch', { @@ -24,7 +24,7 @@ export async function queueProxyFetch(): Promise { export async function queueProxyCheck(proxies: ProxyInfo[]): Promise { const ctx = OperationContext.create('proxy', 'queue-check'); - + const queueManager = QueueManager.getInstance(); const queue = queueManager.getQueue('proxy'); const job = await queue.add('proxy-check', { @@ -37,4 +37,4 @@ export async function queueProxyCheck(proxies: ProxyInfo[]): Promise { const jobId = job.id || 'unknown'; ctx.logger.info('Proxy check job queued', { jobId, count: proxies.length }); return jobId; -} \ No newline at end of file +} diff --git a/apps/data-ingestion/src/handlers/proxy/proxy.handler.ts b/apps/data-ingestion/src/handlers/proxy/proxy.handler.ts index e2e57fa..c98070c 100644 --- a/apps/data-ingestion/src/handlers/proxy/proxy.handler.ts +++ b/apps/data-ingestion/src/handlers/proxy/proxy.handler.ts @@ -1,10 +1,14 @@ /** * Proxy Provider for new queue system */ -import type { ProxyInfo } from '@stock-bot/proxy'; -import { getLogger } from '@stock-bot/logger'; -import { handlerRegistry, createJobHandler, type HandlerConfigWithSchedule } from '@stock-bot/queue'; import type { ServiceContainer } from '@stock-bot/di'; +import { getLogger } from '@stock-bot/logger'; +import type { ProxyInfo } from '@stock-bot/proxy'; +import { + createJobHandler, + handlerRegistry, + type HandlerConfigWithSchedule, +} from '@stock-bot/queue'; const handlerLogger = getLogger('proxy-handler'); diff --git a/apps/data-ingestion/src/handlers/proxy/shared/config.ts b/apps/data-ingestion/src/handlers/proxy/shared/config.ts index 260605b..06481bb 100644 --- a/apps/data-ingestion/src/handlers/proxy/shared/config.ts +++ b/apps/data-ingestion/src/handlers/proxy/shared/config.ts @@ -137,4 +137,4 @@ export const PROXY_CONFIG = { protocol: 'https', }, ], -}; \ No newline at end of file +}; diff --git a/apps/data-ingestion/src/handlers/proxy/shared/types.ts b/apps/data-ingestion/src/handlers/proxy/shared/types.ts index b28c618..331cb3f 100644 --- a/apps/data-ingestion/src/handlers/proxy/shared/types.ts +++ b/apps/data-ingestion/src/handlers/proxy/shared/types.ts @@ -10,4 +10,4 @@ export interface ProxySource { total?: number; // Optional, used for stats percentWorking?: number; // Optional, used for stats lastChecked?: Date; // Optional, used for stats -} \ No newline at end of file +} diff --git a/apps/data-ingestion/src/handlers/qm/actions/exchanges.action.ts b/apps/data-ingestion/src/handlers/qm/actions/exchanges.action.ts index 101d55d..8e7a04b 100644 --- a/apps/data-ingestion/src/handlers/qm/actions/exchanges.action.ts +++ b/apps/data-ingestion/src/handlers/qm/actions/exchanges.action.ts @@ -6,16 +6,14 @@ import type { IServiceContainer } from '@stock-bot/handlers'; export async function fetchExchanges(services: IServiceContainer): Promise { // Get exchanges from MongoDB - const exchanges = await services.mongodb.collection('qm_exchanges') - .find({}).toArray(); - + const exchanges = await services.mongodb.collection('qm_exchanges').find({}).toArray(); + return exchanges; } export async function getExchangeByCode(services: IServiceContainer, code: string): Promise { // Get specific exchange by code - const exchange = await services.mongodb.collection('qm_exchanges') - .findOne({ code }); - + const exchange = await services.mongodb.collection('qm_exchanges').findOne({ code }); + return exchange; -} \ No newline at end of file +} diff --git a/apps/data-ingestion/src/handlers/qm/actions/session.action.ts b/apps/data-ingestion/src/handlers/qm/actions/session.action.ts index 6237c87..ff0df45 100644 --- a/apps/data-ingestion/src/handlers/qm/actions/session.action.ts +++ b/apps/data-ingestion/src/handlers/qm/actions/session.action.ts @@ -9,10 +9,10 @@ import { QMSessionManager } from '../shared/session-manager'; /** * Check existing sessions and queue creation jobs for needed sessions */ -export async function checkSessions(handler: BaseHandler): Promise<{ - cleaned: number; - queued: number; - message: string; +export async function checkSessions(handler: BaseHandler): Promise<{ + cleaned: number; + queued: number; + message: string; }> { const sessionManager = QMSessionManager.getInstance(); const cleanedCount = sessionManager.cleanupFailedSessions(); @@ -24,17 +24,17 @@ export async function checkSessions(handler: BaseHandler): Promise<{ const currentCount = sessionManager.getSessions(sessionId).length; const neededSessions = SESSION_CONFIG.MAX_SESSIONS - currentCount; for (let i = 0; i < neededSessions; i++) { - await handler.scheduleOperation('create-session', { sessionId , sessionType }); + await handler.scheduleOperation('create-session', { sessionId, sessionType }); handler.logger.info(`Queued job to create session for ${sessionType}`); queuedCount++; } } } - + return { cleaned: cleanedCount, queued: queuedCount, - message: `Session check completed: cleaned ${cleanedCount}, queued ${queuedCount}` + message: `Session check completed: cleaned ${cleanedCount}, queued ${queuedCount}`, }; } @@ -42,16 +42,15 @@ export async function checkSessions(handler: BaseHandler): Promise<{ * Create a single session for a specific session ID */ export async function createSingleSession( - handler: BaseHandler, + handler: BaseHandler, input: any ): Promise<{ sessionId: string; status: string; sessionType: string }> { - const { sessionId, sessionType } = input || {}; const sessionManager = QMSessionManager.getInstance(); - + // Get proxy from proxy service const proxyString = handler.proxy.getProxy(); - + // const session = { // proxy: proxyString || 'http://proxy:8080', // headers: sessionManager.getQmHeaders(), @@ -60,15 +59,14 @@ export async function createSingleSession( // lastUsed: new Date() // }; - handler.logger.info(`Creating session for ${sessionType}`) - + handler.logger.info(`Creating session for ${sessionType}`); + // Add session to manager // sessionManager.addSession(sessionType, session); - + return { sessionId: sessionType, status: 'created', - sessionType + sessionType, }; } - diff --git a/apps/data-ingestion/src/handlers/qm/actions/spider.action.ts b/apps/data-ingestion/src/handlers/qm/actions/spider.action.ts index 4dc41ce..2e694c5 100644 --- a/apps/data-ingestion/src/handlers/qm/actions/spider.action.ts +++ b/apps/data-ingestion/src/handlers/qm/actions/spider.action.ts @@ -9,16 +9,15 @@ export async function spiderSymbolSearch( services: IServiceContainer, config: SymbolSpiderJob ): Promise<{ foundSymbols: number; depth: number }> { - // Simple spider implementation // TODO: Implement actual API calls to discover symbols - + // For now, just return mock results const foundSymbols = Math.floor(Math.random() * 10) + 1; - + return { foundSymbols, - depth: config.depth + depth: config.depth, }; } @@ -31,4 +30,4 @@ export async function queueSymbolDiscovery( // TODO: Queue actual discovery jobs await services.cache.set(`discovery:${term}`, { queued: true }, 3600); } -} \ No newline at end of file +} diff --git a/apps/data-ingestion/src/handlers/qm/actions/symbols.action.ts b/apps/data-ingestion/src/handlers/qm/actions/symbols.action.ts index 311ec57..54f004c 100644 --- a/apps/data-ingestion/src/handlers/qm/actions/symbols.action.ts +++ b/apps/data-ingestion/src/handlers/qm/actions/symbols.action.ts @@ -6,16 +6,14 @@ import type { IServiceContainer } from '@stock-bot/handlers'; export async function searchSymbols(services: IServiceContainer): Promise { // Get symbols from MongoDB - const symbols = await services.mongodb.collection('qm_symbols') - .find({}).limit(50).toArray(); - + const symbols = await services.mongodb.collection('qm_symbols').find({}).limit(50).toArray(); + return symbols; } export async function fetchSymbolData(services: IServiceContainer, symbol: string): Promise { // Fetch data for a specific symbol - const symbolData = await services.mongodb.collection('qm_symbols') - .findOne({ symbol }); - + const symbolData = await services.mongodb.collection('qm_symbols').findOne({ symbol }); + return symbolData; -} \ No newline at end of file +} diff --git a/apps/data-ingestion/src/handlers/qm/qm.handler.ts b/apps/data-ingestion/src/handlers/qm/qm.handler.ts index 9b6b7e4..378b700 100644 --- a/apps/data-ingestion/src/handlers/qm/qm.handler.ts +++ b/apps/data-ingestion/src/handlers/qm/qm.handler.ts @@ -1,8 +1,4 @@ -import { - BaseHandler, - Handler, - type IServiceContainer -} from '@stock-bot/handlers'; +import { BaseHandler, Handler, type IServiceContainer } from '@stock-bot/handlers'; @Handler('qm') export class QMHandler extends BaseHandler { @@ -11,10 +7,10 @@ export class QMHandler extends BaseHandler { } // @Operation('check-sessions') - // @QueueSchedule('0 */15 * * *', { - // priority: 7, - // immediately: true, - // description: 'Check and maintain QM sessions' + // @QueueSchedule('0 */15 * * *', { + // priority: 7, + // immediately: true, + // description: 'Check and maintain QM sessions' // }) // async checkSessions(input: unknown, context: ExecutionContext): Promise { // // Call the session maintenance action @@ -36,13 +32,13 @@ export class QMHandler extends BaseHandler { // // Check existing symbols in MongoDB // const symbolsCollection = this.mongodb.collection('qm_symbols'); // const symbols = await symbolsCollection.find({}).limit(100).toArray(); - + // this.logger.info('QM symbol search completed', { count: symbols.length }); - + // if (symbols && symbols.length > 0) { // // Cache result for performance // await this.cache.set('qm-symbols-sample', symbols.slice(0, 10), 1800); - + // return { // success: true, // message: 'QM symbol search completed successfully', @@ -58,7 +54,7 @@ export class QMHandler extends BaseHandler { // count: 0, // }; // } - + // } catch (error) { // this.logger.error('Failed to search QM symbols', { error }); // throw error; @@ -66,10 +62,10 @@ export class QMHandler extends BaseHandler { // } // @Operation('spider-symbol-search') - // @QueueSchedule('0 0 * * 0', { - // priority: 10, - // immediately: false, - // description: 'Comprehensive symbol search using QM API' + // @QueueSchedule('0 0 * * 0', { + // priority: 10, + // immediately: false, + // description: 'Comprehensive symbol search using QM API' // }) // async spiderSymbolSearch(payload: SymbolSpiderJob | undefined, context: ExecutionContext): Promise { // // Set default payload for scheduled runs @@ -79,9 +75,9 @@ export class QMHandler extends BaseHandler { // source: 'qm', // maxDepth: 4 // }; - + // this.logger.info('Starting QM spider symbol search', { payload: jobPayload }); - + // // Store spider job info in cache (temporary data) // const spiderJobId = `spider:qm:${Date.now()}:${Math.random().toString(36).substr(2, 9)}`; // const spiderResult = { @@ -90,19 +86,18 @@ export class QMHandler extends BaseHandler { // status: 'started', // jobId: spiderJobId // }; - + // // Store in cache with 1 hour TTL (temporary data) // await this.cache.set(spiderJobId, spiderResult, 3600); // this.logger.debug('Spider job stored in cache', { spiderJobId, ttl: 3600 }); - + // // Schedule follow-up processing if needed // await this.scheduleOperation('search-symbols', { source: 'spider', spiderJobId }, 5000); - - // return { - // success: true, + + // return { + // success: true, // message: 'QM spider search initiated', // spiderJobId // }; // } - } diff --git a/apps/data-ingestion/src/handlers/qm/shared/config.ts b/apps/data-ingestion/src/handlers/qm/shared/config.ts index 4b5212e..9964359 100644 --- a/apps/data-ingestion/src/handlers/qm/shared/config.ts +++ b/apps/data-ingestion/src/handlers/qm/shared/config.ts @@ -2,11 +2,10 @@ * Shared configuration for QM operations */ - // QM Session IDs for different endpoints export const QM_SESSION_IDS = { LOOKUP: 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6', // lookup endpoint - // '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9': [], //4488d072b + // '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9': [], //4488d072b // cc1cbdaf040f76db8f4c94f7d156b9b9b716e1a7509ec9c74a48a47f6b6b9f87: [], //97ff00cf3 // getQuotes // '74963ff42f1db2320d051762b5d3950ff9eab23f9d5c5b592551b4ca0441d086': [], //32ca24e394b // getSplitsBySymbol getBrokerRatingsBySymbol getDividendsBySymbol getEarningsSurprisesBySymbol getEarningsEventsBySymbol // '1e1d7cb1de1fd2fe52684abdea41a446919a5fe12776dfab88615ac1ce1ec2f6': [], //fb5721812d2c // getEnhancedQuotes getProfiles @@ -36,4 +35,4 @@ export const SESSION_CONFIG = { MAX_FAILED_CALLS: 10, SESSION_TIMEOUT: 10000, // 10 seconds API_TIMEOUT: 15000, // 15 seconds -} as const; \ No newline at end of file +} as const; diff --git a/apps/data-ingestion/src/handlers/qm/shared/session-manager.ts b/apps/data-ingestion/src/handlers/qm/shared/session-manager.ts index 723dcec..ce8d464 100644 --- a/apps/data-ingestion/src/handlers/qm/shared/session-manager.ts +++ b/apps/data-ingestion/src/handlers/qm/shared/session-manager.ts @@ -33,13 +33,15 @@ export class QMSessionManager { if (!sessions || sessions.length === 0) { return null; } - + // Filter out sessions with excessive failures - const validSessions = sessions.filter(session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS); + const validSessions = sessions.filter( + session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS + ); if (validSessions.length === 0) { return null; } - + return validSessions[Math.floor(Math.random() * validSessions.length)]; } @@ -72,7 +74,7 @@ export class QMSessionManager { */ cleanupFailedSessions(): number { let removedCount = 0; - + Object.keys(this.sessionCache).forEach(sessionId => { const initialCount = this.sessionCache[sessionId].length; this.sessionCache[sessionId] = this.sessionCache[sessionId].filter( @@ -80,7 +82,7 @@ export class QMSessionManager { ); removedCount += initialCount - this.sessionCache[sessionId].length; }); - + return removedCount; } @@ -94,13 +96,15 @@ export class QMSessionManager { Referer: 'https://www.quotemedia.com/', }; } - + /** * Check if more sessions are needed for a session ID */ needsMoreSessions(sessionId: string): boolean { const sessions = this.sessionCache[sessionId] || []; - const validSessions = sessions.filter(session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS); + const validSessions = sessions.filter( + session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS + ); return validSessions.length < SESSION_CONFIG.MIN_SESSIONS; } @@ -117,18 +121,22 @@ export class QMSessionManager { */ getStats() { const stats: Record = {}; - + Object.entries(this.sessionCache).forEach(([sessionId, sessions]) => { - const validSessions = sessions.filter(session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS); - const failedSessions = sessions.filter(session => session.failedCalls > SESSION_CONFIG.MAX_FAILED_CALLS); - + const validSessions = sessions.filter( + session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS + ); + const failedSessions = sessions.filter( + session => session.failedCalls > SESSION_CONFIG.MAX_FAILED_CALLS + ); + stats[sessionId] = { total: sessions.length, valid: validSessions.length, - failed: failedSessions.length + failed: failedSessions.length, }; }); - + return stats; } @@ -145,4 +153,4 @@ export class QMSessionManager { getInitialized(): boolean { return this.isInitialized; } -} \ No newline at end of file +} diff --git a/apps/data-ingestion/src/handlers/qm/shared/types.ts b/apps/data-ingestion/src/handlers/qm/shared/types.ts index 9897459..1855a0c 100644 --- a/apps/data-ingestion/src/handlers/qm/shared/types.ts +++ b/apps/data-ingestion/src/handlers/qm/shared/types.ts @@ -29,4 +29,4 @@ export interface SpiderResult { success: boolean; symbolsFound: number; jobsCreated: number; -} \ No newline at end of file +} diff --git a/apps/data-ingestion/src/handlers/webshare/operations/fetch.operations.ts b/apps/data-ingestion/src/handlers/webshare/operations/fetch.operations.ts index ee40fec..aca7037 100644 --- a/apps/data-ingestion/src/handlers/webshare/operations/fetch.operations.ts +++ b/apps/data-ingestion/src/handlers/webshare/operations/fetch.operations.ts @@ -1,9 +1,8 @@ /** * WebShare Fetch Operations - API integration */ -import type { ProxyInfo } from '@stock-bot/proxy'; import { OperationContext } from '@stock-bot/di'; - +import type { ProxyInfo } from '@stock-bot/proxy'; import { WEBSHARE_CONFIG } from '../shared/config'; /** @@ -11,7 +10,7 @@ import { WEBSHARE_CONFIG } from '../shared/config'; */ export async function fetchWebShareProxies(): Promise { const ctx = OperationContext.create('webshare', 'fetch-proxies'); - + try { // Get configuration from config system const { getConfig } = await import('@stock-bot/config'); @@ -30,14 +29,17 @@ export async function fetchWebShareProxies(): Promise { ctx.logger.info('Fetching proxies from WebShare API', { apiUrl }); - const response = await fetch(`${apiUrl}proxy/list/?mode=${WEBSHARE_CONFIG.DEFAULT_MODE}&page=${WEBSHARE_CONFIG.DEFAULT_PAGE}&page_size=${WEBSHARE_CONFIG.DEFAULT_PAGE_SIZE}`, { - method: 'GET', - headers: { - Authorization: `Token ${apiKey}`, - 'Content-Type': 'application/json', - }, - signal: AbortSignal.timeout(WEBSHARE_CONFIG.TIMEOUT), - }); + const response = await fetch( + `${apiUrl}proxy/list/?mode=${WEBSHARE_CONFIG.DEFAULT_MODE}&page=${WEBSHARE_CONFIG.DEFAULT_PAGE}&page_size=${WEBSHARE_CONFIG.DEFAULT_PAGE_SIZE}`, + { + method: 'GET', + headers: { + Authorization: `Token ${apiKey}`, + 'Content-Type': 'application/json', + }, + signal: AbortSignal.timeout(WEBSHARE_CONFIG.TIMEOUT), + } + ); if (!response.ok) { ctx.logger.error('WebShare API request failed', { @@ -55,22 +57,19 @@ export async function fetchWebShareProxies(): Promise { } // Transform proxy data to ProxyInfo format - const proxies: ProxyInfo[] = data.results.map((proxy: { - username: string; - password: string; - proxy_address: string; - port: number; - }) => ({ - source: 'webshare', - protocol: 'http' as const, - host: proxy.proxy_address, - port: proxy.port, - username: proxy.username, - password: proxy.password, - isWorking: true, // WebShare provides working proxies - firstSeen: new Date(), - lastChecked: new Date(), - })); + const proxies: ProxyInfo[] = data.results.map( + (proxy: { username: string; password: string; proxy_address: string; port: number }) => ({ + source: 'webshare', + protocol: 'http' as const, + host: proxy.proxy_address, + port: proxy.port, + username: proxy.username, + password: proxy.password, + isWorking: true, // WebShare provides working proxies + firstSeen: new Date(), + lastChecked: new Date(), + }) + ); ctx.logger.info('Successfully fetched proxies from WebShare', { count: proxies.length, @@ -82,4 +81,4 @@ export async function fetchWebShareProxies(): Promise { ctx.logger.error('Failed to fetch proxies from WebShare', { error }); return []; } -} \ No newline at end of file +} diff --git a/apps/data-ingestion/src/handlers/webshare/shared/config.ts b/apps/data-ingestion/src/handlers/webshare/shared/config.ts index f34aa79..4d3ba82 100644 --- a/apps/data-ingestion/src/handlers/webshare/shared/config.ts +++ b/apps/data-ingestion/src/handlers/webshare/shared/config.ts @@ -7,4 +7,4 @@ export const WEBSHARE_CONFIG = { DEFAULT_MODE: 'direct', DEFAULT_PAGE: 1, TIMEOUT: 10000, -}; \ No newline at end of file +}; diff --git a/apps/data-ingestion/src/handlers/webshare/webshare.handler.ts b/apps/data-ingestion/src/handlers/webshare/webshare.handler.ts index 7dd94f5..20ae89f 100644 --- a/apps/data-ingestion/src/handlers/webshare/webshare.handler.ts +++ b/apps/data-ingestion/src/handlers/webshare/webshare.handler.ts @@ -4,7 +4,7 @@ import { Operation, QueueSchedule, type ExecutionContext, - type IServiceContainer + type IServiceContainer, } from '@stock-bot/handlers'; @Handler('webshare') @@ -14,33 +14,45 @@ export class WebShareHandler extends BaseHandler { } @Operation('fetch-proxies') - @QueueSchedule('0 */6 * * *', { - priority: 3, - immediately: true, - description: 'Fetch fresh proxies from WebShare API' + @QueueSchedule('0 */6 * * *', { + priority: 3, + immediately: true, + description: 'Fetch fresh proxies from WebShare API', }) async fetchProxies(_input: unknown, _context: ExecutionContext): Promise { this.logger.info('Fetching proxies from WebShare API'); - + try { const { fetchWebShareProxies } = await import('./operations/fetch.operations'); const proxies = await fetchWebShareProxies(); - + if (proxies.length > 0) { // Update the centralized proxy manager using the injected service + if (!this.proxy) { + this.logger.warn('Proxy manager is not initialized, cannot update proxies'); + return { + success: false, + proxiesUpdated: 0, + error: 'Proxy manager not initialized', + }; + } await this.proxy.updateProxies(proxies); - - this.logger.info('Updated proxy manager with WebShare proxies', { + + this.logger.info('Updated proxy manager with WebShare proxies', { count: proxies.length, workingCount: proxies.filter(p => p.isWorking !== false).length, }); - + // Cache proxy stats for monitoring await this.cache.set('webshare-proxy-count', proxies.length, 3600); - await this.cache.set('webshare-working-count', proxies.filter(p => p.isWorking !== false).length, 3600); + await this.cache.set( + 'webshare-working-count', + proxies.filter(p => p.isWorking !== false).length, + 3600 + ); await this.cache.set('last-webshare-fetch', new Date().toISOString(), 1800); - - return { + + return { success: true, proxiesUpdated: proxies.length, workingProxies: proxies.filter(p => p.isWorking !== false).length, @@ -59,4 +71,3 @@ export class WebShareHandler extends BaseHandler { } } } - diff --git a/apps/data-ingestion/src/index.ts b/apps/data-ingestion/src/index.ts index 02df141..dddc0d6 100644 --- a/apps/data-ingestion/src/index.ts +++ b/apps/data-ingestion/src/index.ts @@ -4,20 +4,18 @@ */ // Framework imports -import { initializeServiceConfig } from '@stock-bot/config'; import { Hono } from 'hono'; import { cors } from 'hono/cors'; - +import { initializeServiceConfig } from '@stock-bot/config'; // Library imports import { createServiceContainer, initializeServices as initializeAwilixServices, - type ServiceContainer + type ServiceContainer, } from '@stock-bot/di'; import { getLogger, setLoggerConfig, shutdownLoggers } from '@stock-bot/logger'; import { Shutdown } from '@stock-bot/shutdown'; import { handlerRegistry } from '@stock-bot/types'; - // Local imports import { createRoutes } from './routes/create-routes'; import { initializeAllHandlers } from './handlers'; @@ -84,17 +82,17 @@ async function initializeServices() { ttl: 3600, }, }; - + container = createServiceContainer(awilixConfig); await initializeAwilixServices(container); logger.info('Awilix container created and initialized'); - + // Get the service container for handlers const serviceContainer = container.resolve('serviceContainer'); - + // Create app with routes app = new Hono(); - + // Add CORS middleware app.use( '*', @@ -105,17 +103,17 @@ async function initializeServices() { credentials: false, }) ); - + // Create and mount routes using the service container const routes = createRoutes(serviceContainer); app.route('/', routes); // Initialize handlers with service container from Awilix logger.debug('Initializing data handlers with Awilix DI pattern...'); - + // Auto-register all handlers with the service container from Awilix await initializeAllHandlers(serviceContainer); - + logger.info('Data handlers initialized with new DI pattern'); // Create scheduled jobs from registered handlers @@ -175,10 +173,10 @@ async function initializeServices() { logger.info('All services initialized successfully'); } catch (error) { console.error('DETAILED ERROR:', error); - logger.error('Failed to initialize services', { + logger.error('Failed to initialize services', { error: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined, - details: JSON.stringify(error, null, 2) + details: JSON.stringify(error, null, 2), }); throw error; } @@ -236,14 +234,20 @@ shutdown.onShutdownMedium(async () => { if (container) { // Disconnect database clients const mongoClient = container.resolve('mongoClient'); - if (mongoClient?.disconnect) await mongoClient.disconnect(); - + if (mongoClient?.disconnect) { + await mongoClient.disconnect(); + } + const postgresClient = container.resolve('postgresClient'); - if (postgresClient?.disconnect) await postgresClient.disconnect(); - + if (postgresClient?.disconnect) { + await postgresClient.disconnect(); + } + const questdbClient = container.resolve('questdbClient'); - if (questdbClient?.disconnect) await questdbClient.disconnect(); - + if (questdbClient?.disconnect) { + await questdbClient.disconnect(); + } + logger.info('All services disposed successfully'); } } catch (error) { @@ -268,4 +272,4 @@ startServer().catch(error => { process.exit(1); }); -logger.info('Data service startup initiated with improved DI pattern'); \ No newline at end of file +logger.info('Data service startup initiated with improved DI pattern'); diff --git a/apps/data-ingestion/src/routes/create-routes.ts b/apps/data-ingestion/src/routes/create-routes.ts index 5e19fee..7c4a341 100644 --- a/apps/data-ingestion/src/routes/create-routes.ts +++ b/apps/data-ingestion/src/routes/create-routes.ts @@ -1,69 +1,74 @@ -/** - * Routes creation with improved DI pattern - */ - -import { Hono } from 'hono'; -import type { IServiceContainer } from '@stock-bot/handlers'; -import { exchangeRoutes } from './exchange.routes'; -import { healthRoutes } from './health.routes'; -import { queueRoutes } from './queue.routes'; - -/** - * Creates all routes with access to type-safe services - */ -export function createRoutes(services: IServiceContainer): Hono { - const app = new Hono(); - - // Mount routes that don't need services - app.route('/health', healthRoutes); - - // Mount routes that need services (will be updated to use services) - app.route('/api/exchanges', exchangeRoutes); - app.route('/api/queue', queueRoutes); - - // Store services in app context for handlers that need it - app.use('*', async (c, next) => { - c.set('services', services); - await next(); - }); - - // Add a new endpoint to test the improved DI - app.get('/api/di-test', async (c) => { - try { - const services = c.get('services') as IServiceContainer; - - // Test MongoDB connection - const mongoStats = services.mongodb?.getPoolMetrics?.() || { status: services.mongodb ? 'connected' : 'disabled' }; - - // Test PostgreSQL connection - const pgConnected = services.postgres?.connected || false; - - // Test cache - const cacheReady = services.cache?.isReady() || false; - - // Test queue - const queueStats = services.queue?.getGlobalStats() || { status: 'disabled' }; - - return c.json({ - success: true, - message: 'Improved DI pattern is working!', - services: { - mongodb: mongoStats, - postgres: { connected: pgConnected }, - cache: { ready: cacheReady }, - queue: queueStats - }, - timestamp: new Date().toISOString() - }); - } catch (error) { - const services = c.get('services') as IServiceContainer; - services.logger.error('DI test endpoint failed', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : String(error) - }, 500); - } - }); - - return app; -} \ No newline at end of file +/** + * Routes creation with improved DI pattern + */ + +import { Hono } from 'hono'; +import type { IServiceContainer } from '@stock-bot/handlers'; +import { exchangeRoutes } from './exchange.routes'; +import { healthRoutes } from './health.routes'; +import { queueRoutes } from './queue.routes'; + +/** + * Creates all routes with access to type-safe services + */ +export function createRoutes(services: IServiceContainer): Hono { + const app = new Hono(); + + // Mount routes that don't need services + app.route('/health', healthRoutes); + + // Mount routes that need services (will be updated to use services) + app.route('/api/exchanges', exchangeRoutes); + app.route('/api/queue', queueRoutes); + + // Store services in app context for handlers that need it + app.use('*', async (c, next) => { + c.set('services', services); + await next(); + }); + + // Add a new endpoint to test the improved DI + app.get('/api/di-test', async c => { + try { + const services = c.get('services') as IServiceContainer; + + // Test MongoDB connection + const mongoStats = services.mongodb?.getPoolMetrics?.() || { + status: services.mongodb ? 'connected' : 'disabled', + }; + + // Test PostgreSQL connection + const pgConnected = services.postgres?.connected || false; + + // Test cache + const cacheReady = services.cache?.isReady() || false; + + // Test queue + const queueStats = services.queue?.getGlobalStats() || { status: 'disabled' }; + + return c.json({ + success: true, + message: 'Improved DI pattern is working!', + services: { + mongodb: mongoStats, + postgres: { connected: pgConnected }, + cache: { ready: cacheReady }, + queue: queueStats, + }, + timestamp: new Date().toISOString(), + }); + } catch (error) { + const services = c.get('services') as IServiceContainer; + services.logger.error('DI test endpoint failed', { error }); + return c.json( + { + success: false, + error: error instanceof Error ? error.message : String(error), + }, + 500 + ); + } + }); + + return app; +} diff --git a/apps/data-ingestion/src/routes/exchange.routes.ts b/apps/data-ingestion/src/routes/exchange.routes.ts index 1752270..4083fd5 100644 --- a/apps/data-ingestion/src/routes/exchange.routes.ts +++ b/apps/data-ingestion/src/routes/exchange.routes.ts @@ -11,7 +11,7 @@ exchange.get('/', async c => { return c.json({ status: 'success', data: [], - message: 'Exchange endpoints will be implemented with database integration' + message: 'Exchange endpoints will be implemented with database integration', }); } catch (error) { logger.error('Failed to get exchanges', { error }); @@ -19,4 +19,4 @@ exchange.get('/', async c => { } }); -export { exchange as exchangeRoutes }; \ No newline at end of file +export { exchange as exchangeRoutes }; diff --git a/apps/data-ingestion/src/routes/queue.routes.ts b/apps/data-ingestion/src/routes/queue.routes.ts index 20a8d4d..8dd8edc 100644 --- a/apps/data-ingestion/src/routes/queue.routes.ts +++ b/apps/data-ingestion/src/routes/queue.routes.ts @@ -10,11 +10,11 @@ queue.get('/status', async c => { try { const queueManager = QueueManager.getInstance(); const globalStats = await queueManager.getGlobalStats(); - + return c.json({ status: 'success', data: globalStats, - message: 'Queue status retrieved successfully' + message: 'Queue status retrieved successfully', }); } catch (error) { logger.error('Failed to get queue status', { error }); @@ -22,4 +22,4 @@ queue.get('/status', async c => { } }); -export { queue as queueRoutes }; \ No newline at end of file +export { queue as queueRoutes }; diff --git a/apps/data-ingestion/src/types/exchange.types.ts b/apps/data-ingestion/src/types/exchange.types.ts index 13a10b3..6fcbdd8 100644 --- a/apps/data-ingestion/src/types/exchange.types.ts +++ b/apps/data-ingestion/src/types/exchange.types.ts @@ -37,4 +37,4 @@ export interface IBSymbol { name?: string; currency?: string; // Add other properties as needed -} \ No newline at end of file +} diff --git a/apps/data-ingestion/src/types/job-payloads.ts b/apps/data-ingestion/src/types/job-payloads.ts index af2f82c..05144b8 100644 --- a/apps/data-ingestion/src/types/job-payloads.ts +++ b/apps/data-ingestion/src/types/job-payloads.ts @@ -90,4 +90,4 @@ export interface FetchWebShareProxiesResult extends CountableJobResult { // No payload job types (for operations that don't need input) export interface NoPayload { // Empty interface for operations that don't need payload -} \ No newline at end of file +} diff --git a/apps/data-ingestion/src/utils/symbol-search.util.ts b/apps/data-ingestion/src/utils/symbol-search.util.ts index 7e2379a..fd40f1c 100644 --- a/apps/data-ingestion/src/utils/symbol-search.util.ts +++ b/apps/data-ingestion/src/utils/symbol-search.util.ts @@ -1,5 +1,5 @@ -import { getLogger } from '@stock-bot/logger'; import { sleep } from '@stock-bot/di'; +import { getLogger } from '@stock-bot/logger'; const logger = getLogger('symbol-search-util'); diff --git a/apps/data-ingestion/test-ceo-operations.ts b/apps/data-ingestion/test-ceo-operations.ts index 9250e72..7a46724 100755 --- a/apps/data-ingestion/test-ceo-operations.ts +++ b/apps/data-ingestion/test-ceo-operations.ts @@ -1,101 +1,103 @@ -#!/usr/bin/env bun - -/** - * Test script for CEO handler operations - */ - -import { initializeServiceConfig } from '@stock-bot/config'; -import { createServiceContainer, initializeServices } from '@stock-bot/di'; -import { getLogger } from '@stock-bot/logger'; - -const logger = getLogger('test-ceo-operations'); - -async function testCeoOperations() { - logger.info('Testing CEO handler operations...'); - - try { - // Initialize config - const config = initializeServiceConfig(); - - // Create Awilix container - const awilixConfig = { - redis: { - host: config.database.dragonfly.host, - port: config.database.dragonfly.port, - db: config.database.dragonfly.db, - }, - mongodb: { - uri: config.database.mongodb.uri, - database: config.database.mongodb.database, - }, - postgres: { - host: config.database.postgres.host, - port: config.database.postgres.port, - database: config.database.postgres.database, - user: config.database.postgres.user, - password: config.database.postgres.password, - }, - questdb: { - enabled: false, - host: config.database.questdb.host, - httpPort: config.database.questdb.httpPort, - pgPort: config.database.questdb.pgPort, - influxPort: config.database.questdb.ilpPort, - database: config.database.questdb.database, - }, - }; - - const container = createServiceContainer(awilixConfig); - await initializeServices(container); - - const serviceContainer = container.resolve('serviceContainer'); - - // Import and create CEO handler - const { CeoHandler } = await import('./src/handlers/ceo/ceo.handler'); - const ceoHandler = new CeoHandler(serviceContainer); - - // Test 1: Check if there are any CEO symbols in the database - logger.info('Checking for existing CEO symbols...'); - const collection = serviceContainer.mongodb.collection('ceoSymbols'); - const count = await collection.countDocuments(); - logger.info(`Found ${count} CEO symbols in database`); - - if (count > 0) { - // Test 2: Run process-unique-symbols operation - logger.info('Testing process-unique-symbols operation...'); - const result = await ceoHandler.updateUniqueSymbols(undefined, {}); - logger.info('Process unique symbols result:', result); - - // Test 3: Test individual symbol processing - logger.info('Testing process-individual-symbol operation...'); - const sampleSymbol = await collection.findOne({}); - if (sampleSymbol) { - const individualResult = await ceoHandler.processIndividualSymbol({ - ceoId: sampleSymbol.ceoId, - symbol: sampleSymbol.symbol, - exchange: sampleSymbol.exchange, - name: sampleSymbol.name, - }, {}); - logger.info('Process individual symbol result:', individualResult); - } - } else { - logger.warn('No CEO symbols found. Run the service to populate data first.'); - } - - // Clean up - await serviceContainer.mongodb.disconnect(); - await serviceContainer.postgres.disconnect(); - if (serviceContainer.cache) { - await serviceContainer.cache.disconnect(); - } - - logger.info('Test completed successfully!'); - process.exit(0); - } catch (error) { - logger.error('Test failed:', error); - process.exit(1); - } -} - -// Run the test -testCeoOperations(); \ No newline at end of file +#!/usr/bin/env bun + +/** + * Test script for CEO handler operations + */ +import { initializeServiceConfig } from '@stock-bot/config'; +import { createServiceContainer, initializeServices } from '@stock-bot/di'; +import { getLogger } from '@stock-bot/logger'; + +const logger = getLogger('test-ceo-operations'); + +async function testCeoOperations() { + logger.info('Testing CEO handler operations...'); + + try { + // Initialize config + const config = initializeServiceConfig(); + + // Create Awilix container + const awilixConfig = { + redis: { + host: config.database.dragonfly.host, + port: config.database.dragonfly.port, + db: config.database.dragonfly.db, + }, + mongodb: { + uri: config.database.mongodb.uri, + database: config.database.mongodb.database, + }, + postgres: { + host: config.database.postgres.host, + port: config.database.postgres.port, + database: config.database.postgres.database, + user: config.database.postgres.user, + password: config.database.postgres.password, + }, + questdb: { + enabled: false, + host: config.database.questdb.host, + httpPort: config.database.questdb.httpPort, + pgPort: config.database.questdb.pgPort, + influxPort: config.database.questdb.ilpPort, + database: config.database.questdb.database, + }, + }; + + const container = createServiceContainer(awilixConfig); + await initializeServices(container); + + const serviceContainer = container.resolve('serviceContainer'); + + // Import and create CEO handler + const { CeoHandler } = await import('./src/handlers/ceo/ceo.handler'); + const ceoHandler = new CeoHandler(serviceContainer); + + // Test 1: Check if there are any CEO symbols in the database + logger.info('Checking for existing CEO symbols...'); + const collection = serviceContainer.mongodb.collection('ceoSymbols'); + const count = await collection.countDocuments(); + logger.info(`Found ${count} CEO symbols in database`); + + if (count > 0) { + // Test 2: Run process-unique-symbols operation + logger.info('Testing process-unique-symbols operation...'); + const result = await ceoHandler.updateUniqueSymbols(undefined, {}); + logger.info('Process unique symbols result:', result); + + // Test 3: Test individual symbol processing + logger.info('Testing process-individual-symbol operation...'); + const sampleSymbol = await collection.findOne({}); + if (sampleSymbol) { + const individualResult = await ceoHandler.processIndividualSymbol( + { + ceoId: sampleSymbol.ceoId, + symbol: sampleSymbol.symbol, + exchange: sampleSymbol.exchange, + name: sampleSymbol.name, + }, + {} + ); + logger.info('Process individual symbol result:', individualResult); + } + } else { + logger.warn('No CEO symbols found. Run the service to populate data first.'); + } + + // Clean up + await serviceContainer.mongodb.disconnect(); + await serviceContainer.postgres.disconnect(); + if (serviceContainer.cache) { + await serviceContainer.cache.disconnect(); + } + + logger.info('Test completed successfully!'); + process.exit(0); + } catch (error) { + logger.error('Test failed:', error); + process.exit(1); + } +} + +// Run the test +testCeoOperations(); diff --git a/apps/data-pipeline/config/default.json b/apps/data-pipeline/config/default.json index a886025..a7893f4 100644 --- a/apps/data-pipeline/config/default.json +++ b/apps/data-pipeline/config/default.json @@ -1,15 +1,15 @@ -{ - "service": { - "name": "data-pipeline", - "port": 3005, - "host": "0.0.0.0", - "healthCheckPath": "/health", - "metricsPath": "/metrics", - "shutdownTimeout": 30000, - "cors": { - "enabled": true, - "origin": "*", - "credentials": false - } - } -} \ No newline at end of file +{ + "service": { + "name": "data-pipeline", + "port": 3005, + "host": "0.0.0.0", + "healthCheckPath": "/health", + "metricsPath": "/metrics", + "shutdownTimeout": 30000, + "cors": { + "enabled": true, + "origin": "*", + "credentials": false + } + } +} diff --git a/apps/data-pipeline/src/clients.ts b/apps/data-pipeline/src/clients.ts index 4bc8217..8cd54e2 100644 --- a/apps/data-pipeline/src/clients.ts +++ b/apps/data-pipeline/src/clients.ts @@ -1,27 +1,27 @@ -import { PostgreSQLClient } from '@stock-bot/postgres'; -import { MongoDBClient } from '@stock-bot/mongodb'; - -let postgresClient: PostgreSQLClient | null = null; -let mongodbClient: MongoDBClient | null = null; - -export function setPostgreSQLClient(client: PostgreSQLClient): void { - postgresClient = client; -} - -export function getPostgreSQLClient(): PostgreSQLClient { - if (!postgresClient) { - throw new Error('PostgreSQL client not initialized. Call setPostgreSQLClient first.'); - } - return postgresClient; -} - -export function setMongoDBClient(client: MongoDBClient): void { - mongodbClient = client; -} - -export function getMongoDBClient(): MongoDBClient { - if (!mongodbClient) { - throw new Error('MongoDB client not initialized. Call setMongoDBClient first.'); - } - return mongodbClient; -} \ No newline at end of file +import { MongoDBClient } from '@stock-bot/mongodb'; +import { PostgreSQLClient } from '@stock-bot/postgres'; + +let postgresClient: PostgreSQLClient | null = null; +let mongodbClient: MongoDBClient | null = null; + +export function setPostgreSQLClient(client: PostgreSQLClient): void { + postgresClient = client; +} + +export function getPostgreSQLClient(): PostgreSQLClient { + if (!postgresClient) { + throw new Error('PostgreSQL client not initialized. Call setPostgreSQLClient first.'); + } + return postgresClient; +} + +export function setMongoDBClient(client: MongoDBClient): void { + mongodbClient = client; +} + +export function getMongoDBClient(): MongoDBClient { + if (!mongodbClient) { + throw new Error('MongoDB client not initialized. Call setMongoDBClient first.'); + } + return mongodbClient; +} diff --git a/apps/data-pipeline/src/handlers/exchanges/exchanges.handler.ts b/apps/data-pipeline/src/handlers/exchanges/exchanges.handler.ts index 06aa283..2968dd2 100644 --- a/apps/data-pipeline/src/handlers/exchanges/exchanges.handler.ts +++ b/apps/data-pipeline/src/handlers/exchanges/exchanges.handler.ts @@ -1,58 +1,58 @@ -import { getLogger } from '@stock-bot/logger'; -import { handlerRegistry, type HandlerConfig, type ScheduledJobConfig } from '@stock-bot/queue'; -import { exchangeOperations } from './operations'; - -const logger = getLogger('exchanges-handler'); - -const HANDLER_NAME = 'exchanges'; - -const exchangesHandlerConfig: HandlerConfig = { - concurrency: 1, - maxAttempts: 3, - scheduledJobs: [ - { - operation: 'sync-all-exchanges', - cronPattern: '0 0 * * 0', // Weekly on Sunday at midnight - payload: { clearFirst: true }, - priority: 10, - immediately: false, - } as ScheduledJobConfig, - { - operation: 'sync-qm-exchanges', - cronPattern: '0 1 * * *', // Daily at 1 AM - payload: {}, - priority: 5, - immediately: false, - } as ScheduledJobConfig, - { - operation: 'sync-ib-exchanges', - cronPattern: '0 3 * * *', // Daily at 3 AM - payload: {}, - priority: 3, - immediately: false, - } as ScheduledJobConfig, - { - operation: 'sync-qm-provider-mappings', - cronPattern: '0 3 * * *', // Daily at 3 AM - payload: {}, - priority: 7, - immediately: false, - } as ScheduledJobConfig, - ], - operations: { - 'sync-all-exchanges': exchangeOperations.syncAllExchanges, - 'sync-qm-exchanges': exchangeOperations.syncQMExchanges, - 'sync-ib-exchanges': exchangeOperations.syncIBExchanges, - 'sync-qm-provider-mappings': exchangeOperations.syncQMProviderMappings, - 'clear-postgresql-data': exchangeOperations.clearPostgreSQLData, - 'get-exchange-stats': exchangeOperations.getExchangeStats, - 'get-provider-mapping-stats': exchangeOperations.getProviderMappingStats, - 'enhanced-sync-status': exchangeOperations['enhanced-sync-status'], - }, -}; - -export function initializeExchangesHandler(): void { - logger.info('Registering exchanges handler...'); - handlerRegistry.registerHandler(HANDLER_NAME, exchangesHandlerConfig); - logger.info('Exchanges handler registered successfully'); -} \ No newline at end of file +import { getLogger } from '@stock-bot/logger'; +import { handlerRegistry, type HandlerConfig, type ScheduledJobConfig } from '@stock-bot/queue'; +import { exchangeOperations } from './operations'; + +const logger = getLogger('exchanges-handler'); + +const HANDLER_NAME = 'exchanges'; + +const exchangesHandlerConfig: HandlerConfig = { + concurrency: 1, + maxAttempts: 3, + scheduledJobs: [ + { + operation: 'sync-all-exchanges', + cronPattern: '0 0 * * 0', // Weekly on Sunday at midnight + payload: { clearFirst: true }, + priority: 10, + immediately: false, + } as ScheduledJobConfig, + { + operation: 'sync-qm-exchanges', + cronPattern: '0 1 * * *', // Daily at 1 AM + payload: {}, + priority: 5, + immediately: false, + } as ScheduledJobConfig, + { + operation: 'sync-ib-exchanges', + cronPattern: '0 3 * * *', // Daily at 3 AM + payload: {}, + priority: 3, + immediately: false, + } as ScheduledJobConfig, + { + operation: 'sync-qm-provider-mappings', + cronPattern: '0 3 * * *', // Daily at 3 AM + payload: {}, + priority: 7, + immediately: false, + } as ScheduledJobConfig, + ], + operations: { + 'sync-all-exchanges': exchangeOperations.syncAllExchanges, + 'sync-qm-exchanges': exchangeOperations.syncQMExchanges, + 'sync-ib-exchanges': exchangeOperations.syncIBExchanges, + 'sync-qm-provider-mappings': exchangeOperations.syncQMProviderMappings, + 'clear-postgresql-data': exchangeOperations.clearPostgreSQLData, + 'get-exchange-stats': exchangeOperations.getExchangeStats, + 'get-provider-mapping-stats': exchangeOperations.getProviderMappingStats, + 'enhanced-sync-status': exchangeOperations['enhanced-sync-status'], + }, +}; + +export function initializeExchangesHandler(): void { + logger.info('Registering exchanges handler...'); + handlerRegistry.registerHandler(HANDLER_NAME, exchangesHandlerConfig); + logger.info('Exchanges handler registered successfully'); +} diff --git a/apps/data-pipeline/src/handlers/exchanges/operations/clear-postgresql-data.operations.ts b/apps/data-pipeline/src/handlers/exchanges/operations/clear-postgresql-data.operations.ts index d7148ed..808320e 100644 --- a/apps/data-pipeline/src/handlers/exchanges/operations/clear-postgresql-data.operations.ts +++ b/apps/data-pipeline/src/handlers/exchanges/operations/clear-postgresql-data.operations.ts @@ -13,7 +13,7 @@ export async function clearPostgreSQLData(payload: JobPayload): Promise<{ try { const postgresClient = getPostgreSQLClient(); - + // Start transaction for atomic operations await postgresClient.query('BEGIN'); @@ -21,9 +21,7 @@ export async function clearPostgreSQLData(payload: JobPayload): Promise<{ const exchangeCountResult = await postgresClient.query( 'SELECT COUNT(*) as count FROM exchanges' ); - const symbolCountResult = await postgresClient.query( - 'SELECT COUNT(*) as count FROM symbols' - ); + const symbolCountResult = await postgresClient.query('SELECT COUNT(*) as count FROM symbols'); const mappingCountResult = await postgresClient.query( 'SELECT COUNT(*) as count FROM provider_mappings' ); @@ -57,4 +55,4 @@ export async function clearPostgreSQLData(payload: JobPayload): Promise<{ logger.error('Failed to clear PostgreSQL data', { error }); throw error; } -} \ No newline at end of file +} diff --git a/apps/data-pipeline/src/handlers/exchanges/operations/enhanced-sync-status.operations.ts b/apps/data-pipeline/src/handlers/exchanges/operations/enhanced-sync-status.operations.ts index 275ac82..da188e9 100644 --- a/apps/data-pipeline/src/handlers/exchanges/operations/enhanced-sync-status.operations.ts +++ b/apps/data-pipeline/src/handlers/exchanges/operations/enhanced-sync-status.operations.ts @@ -16,11 +16,11 @@ export async function getSyncStatus(payload: JobPayload): Promise ORDER BY provider, data_type `; const result = await postgresClient.query(query); - + logger.info(`Retrieved sync status for ${result.rows.length} entries`); return result.rows; } catch (error) { logger.error('Failed to get sync status', { error }); throw error; } -} \ No newline at end of file +} diff --git a/apps/data-pipeline/src/handlers/exchanges/operations/exchange-stats.operations.ts b/apps/data-pipeline/src/handlers/exchanges/operations/exchange-stats.operations.ts index b67170e..2c79d96 100644 --- a/apps/data-pipeline/src/handlers/exchanges/operations/exchange-stats.operations.ts +++ b/apps/data-pipeline/src/handlers/exchanges/operations/exchange-stats.operations.ts @@ -18,11 +18,11 @@ export async function getExchangeStats(payload: JobPayload): Promise { FROM exchanges `; const result = await postgresClient.query(query); - + logger.info('Retrieved exchange statistics'); return result.rows[0]; } catch (error) { logger.error('Failed to get exchange statistics', { error }); throw error; } -} \ No newline at end of file +} diff --git a/apps/data-pipeline/src/handlers/exchanges/operations/index.ts b/apps/data-pipeline/src/handlers/exchanges/operations/index.ts index b5157d7..b798ee5 100644 --- a/apps/data-pipeline/src/handlers/exchanges/operations/index.ts +++ b/apps/data-pipeline/src/handlers/exchanges/operations/index.ts @@ -1,19 +1,19 @@ -import { syncAllExchanges } from './sync-all-exchanges.operations'; -import { syncQMExchanges } from './qm-exchanges.operations'; -import { syncIBExchanges } from './sync-ib-exchanges.operations'; -import { syncQMProviderMappings } from './sync-qm-provider-mappings.operations'; -import { clearPostgreSQLData } from './clear-postgresql-data.operations'; -import { getExchangeStats } from './exchange-stats.operations'; -import { getProviderMappingStats } from './provider-mapping-stats.operations'; -import { getSyncStatus } from './enhanced-sync-status.operations'; - -export const exchangeOperations = { - syncAllExchanges, - syncQMExchanges, - syncIBExchanges, - syncQMProviderMappings, - clearPostgreSQLData, - getExchangeStats, - getProviderMappingStats, - 'enhanced-sync-status': getSyncStatus, -}; \ No newline at end of file +import { clearPostgreSQLData } from './clear-postgresql-data.operations'; +import { getSyncStatus } from './enhanced-sync-status.operations'; +import { getExchangeStats } from './exchange-stats.operations'; +import { getProviderMappingStats } from './provider-mapping-stats.operations'; +import { syncQMExchanges } from './qm-exchanges.operations'; +import { syncAllExchanges } from './sync-all-exchanges.operations'; +import { syncIBExchanges } from './sync-ib-exchanges.operations'; +import { syncQMProviderMappings } from './sync-qm-provider-mappings.operations'; + +export const exchangeOperations = { + syncAllExchanges, + syncQMExchanges, + syncIBExchanges, + syncQMProviderMappings, + clearPostgreSQLData, + getExchangeStats, + getProviderMappingStats, + 'enhanced-sync-status': getSyncStatus, +}; diff --git a/apps/data-pipeline/src/handlers/exchanges/operations/provider-mapping-stats.operations.ts b/apps/data-pipeline/src/handlers/exchanges/operations/provider-mapping-stats.operations.ts index 9e04eca..62cb229 100644 --- a/apps/data-pipeline/src/handlers/exchanges/operations/provider-mapping-stats.operations.ts +++ b/apps/data-pipeline/src/handlers/exchanges/operations/provider-mapping-stats.operations.ts @@ -22,11 +22,11 @@ export async function getProviderMappingStats(payload: JobPayload): Promise ORDER BY provider `; const result = await postgresClient.query(query); - + logger.info('Retrieved provider mapping statistics'); return result.rows; } catch (error) { logger.error('Failed to get provider mapping statistics', { error }); throw error; } -} \ No newline at end of file +} diff --git a/apps/data-pipeline/src/handlers/exchanges/operations/qm-exchanges.operations.ts b/apps/data-pipeline/src/handlers/exchanges/operations/qm-exchanges.operations.ts index 13ebe7e..cebea42 100644 --- a/apps/data-pipeline/src/handlers/exchanges/operations/qm-exchanges.operations.ts +++ b/apps/data-pipeline/src/handlers/exchanges/operations/qm-exchanges.operations.ts @@ -1,102 +1,113 @@ -import { getLogger } from '@stock-bot/logger'; -import { getMongoDBClient, getPostgreSQLClient } from '../../../clients'; -import type { JobPayload } from '../../../types/job-payloads'; - -const logger = getLogger('sync-qm-exchanges'); - -export async function syncQMExchanges(payload: JobPayload): Promise<{ processed: number; created: number; updated: number }> { - logger.info('Starting QM exchanges sync...'); - - try { - const mongoClient = getMongoDBClient(); - const postgresClient = getPostgreSQLClient(); - - // 1. Get all QM exchanges from MongoDB - const qmExchanges = await mongoClient.find('qmExchanges', {}); - logger.info(`Found ${qmExchanges.length} QM exchanges to process`); - - let created = 0; - let updated = 0; - - for (const exchange of qmExchanges) { - try { - // 2. Check if exchange exists - const existingExchange = await findExchange(exchange.exchangeCode, postgresClient); - - if (existingExchange) { - // Update existing - await updateExchange(existingExchange.id, exchange, postgresClient); - updated++; - } else { - // Create new - await createExchange(exchange, postgresClient); - created++; - } - } catch (error) { - logger.error('Failed to process exchange', { error, exchange: exchange.exchangeCode }); - } - } - - // 3. Update sync status - await updateSyncStatus('qm', 'exchanges', qmExchanges.length, postgresClient); - - const result = { processed: qmExchanges.length, created, updated }; - logger.info('QM exchanges sync completed', result); - return result; - } catch (error) { - logger.error('QM exchanges sync failed', { error }); - throw error; - } -} - -// Helper functions -async function findExchange(exchangeCode: string, postgresClient: any): Promise { - const query = 'SELECT * FROM exchanges WHERE code = $1'; - const result = await postgresClient.query(query, [exchangeCode]); - return result.rows[0] || null; -} - -async function createExchange(qmExchange: any, postgresClient: any): Promise { - const query = ` - INSERT INTO exchanges (code, name, country, currency, visible) - VALUES ($1, $2, $3, $4, $5) - ON CONFLICT (code) DO NOTHING - `; - - await postgresClient.query(query, [ - qmExchange.exchangeCode || qmExchange.exchange, - qmExchange.exchangeShortName || qmExchange.name, - qmExchange.countryCode || 'US', - 'USD', // Default currency, can be improved - true, // New exchanges are visible by default - ]); -} - -async function updateExchange(exchangeId: string, qmExchange: any, postgresClient: any): Promise { - const query = ` - UPDATE exchanges - SET name = COALESCE($2, name), - country = COALESCE($3, country), - updated_at = NOW() - WHERE id = $1 - `; - - await postgresClient.query(query, [ - exchangeId, - qmExchange.exchangeShortName || qmExchange.name, - qmExchange.countryCode, - ]); -} - -async function updateSyncStatus(provider: string, dataType: string, count: number, postgresClient: any): Promise { - const query = ` - UPDATE sync_status - SET last_sync_at = NOW(), - last_sync_count = $3, - sync_errors = NULL, - updated_at = NOW() - WHERE provider = $1 AND data_type = $2 - `; - - await postgresClient.query(query, [provider, dataType, count]); -} \ No newline at end of file +import { getLogger } from '@stock-bot/logger'; +import { getMongoDBClient, getPostgreSQLClient } from '../../../clients'; +import type { JobPayload } from '../../../types/job-payloads'; + +const logger = getLogger('sync-qm-exchanges'); + +export async function syncQMExchanges( + payload: JobPayload +): Promise<{ processed: number; created: number; updated: number }> { + logger.info('Starting QM exchanges sync...'); + + try { + const mongoClient = getMongoDBClient(); + const postgresClient = getPostgreSQLClient(); + + // 1. Get all QM exchanges from MongoDB + const qmExchanges = await mongoClient.find('qmExchanges', {}); + logger.info(`Found ${qmExchanges.length} QM exchanges to process`); + + let created = 0; + let updated = 0; + + for (const exchange of qmExchanges) { + try { + // 2. Check if exchange exists + const existingExchange = await findExchange(exchange.exchangeCode, postgresClient); + + if (existingExchange) { + // Update existing + await updateExchange(existingExchange.id, exchange, postgresClient); + updated++; + } else { + // Create new + await createExchange(exchange, postgresClient); + created++; + } + } catch (error) { + logger.error('Failed to process exchange', { error, exchange: exchange.exchangeCode }); + } + } + + // 3. Update sync status + await updateSyncStatus('qm', 'exchanges', qmExchanges.length, postgresClient); + + const result = { processed: qmExchanges.length, created, updated }; + logger.info('QM exchanges sync completed', result); + return result; + } catch (error) { + logger.error('QM exchanges sync failed', { error }); + throw error; + } +} + +// Helper functions +async function findExchange(exchangeCode: string, postgresClient: any): Promise { + const query = 'SELECT * FROM exchanges WHERE code = $1'; + const result = await postgresClient.query(query, [exchangeCode]); + return result.rows[0] || null; +} + +async function createExchange(qmExchange: any, postgresClient: any): Promise { + const query = ` + INSERT INTO exchanges (code, name, country, currency, visible) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (code) DO NOTHING + `; + + await postgresClient.query(query, [ + qmExchange.exchangeCode || qmExchange.exchange, + qmExchange.exchangeShortName || qmExchange.name, + qmExchange.countryCode || 'US', + 'USD', // Default currency, can be improved + true, // New exchanges are visible by default + ]); +} + +async function updateExchange( + exchangeId: string, + qmExchange: any, + postgresClient: any +): Promise { + const query = ` + UPDATE exchanges + SET name = COALESCE($2, name), + country = COALESCE($3, country), + updated_at = NOW() + WHERE id = $1 + `; + + await postgresClient.query(query, [ + exchangeId, + qmExchange.exchangeShortName || qmExchange.name, + qmExchange.countryCode, + ]); +} + +async function updateSyncStatus( + provider: string, + dataType: string, + count: number, + postgresClient: any +): Promise { + const query = ` + UPDATE sync_status + SET last_sync_at = NOW(), + last_sync_count = $3, + sync_errors = NULL, + updated_at = NOW() + WHERE provider = $1 AND data_type = $2 + `; + + await postgresClient.query(query, [provider, dataType, count]); +} diff --git a/apps/data-pipeline/src/handlers/exchanges/operations/sync-all-exchanges.operations.ts b/apps/data-pipeline/src/handlers/exchanges/operations/sync-all-exchanges.operations.ts index 636347c..e6ba7fd 100644 --- a/apps/data-pipeline/src/handlers/exchanges/operations/sync-all-exchanges.operations.ts +++ b/apps/data-pipeline/src/handlers/exchanges/operations/sync-all-exchanges.operations.ts @@ -1,266 +1,275 @@ -import { getLogger } from '@stock-bot/logger'; -import { getMongoDBClient, getPostgreSQLClient } from '../../../clients'; -import type { JobPayload, SyncResult } from '../../../types/job-payloads'; - -const logger = getLogger('enhanced-sync-all-exchanges'); - -export async function syncAllExchanges(payload: JobPayload): Promise { - const clearFirst = payload.clearFirst || true; - logger.info('Starting comprehensive exchange sync...', { clearFirst }); - - const result: SyncResult = { - processed: 0, - created: 0, - updated: 0, - skipped: 0, - errors: 0, - }; - - try { - const postgresClient = getPostgreSQLClient(); - - // Clear existing data if requested - if (clearFirst) { - await clearPostgreSQLData(postgresClient); - } - - // Start transaction for atomic operations - await postgresClient.query('BEGIN'); - - // 1. Sync from EOD exchanges (comprehensive global data) - const eodResult = await syncEODExchanges(); - mergeResults(result, eodResult); - - // 2. Sync from IB exchanges (detailed asset information) - const ibResult = await syncIBExchanges(); - mergeResults(result, ibResult); - - // 3. Update sync status - await updateSyncStatus('all', 'exchanges', result.processed, postgresClient); - - await postgresClient.query('COMMIT'); - - logger.info('Comprehensive exchange sync completed', result); - return result; - } catch (error) { - const postgresClient = getPostgreSQLClient(); - await postgresClient.query('ROLLBACK'); - logger.error('Comprehensive exchange sync failed', { error }); - throw error; - } -} - -async function clearPostgreSQLData(postgresClient: any): Promise { - logger.info('Clearing existing PostgreSQL data...'); - - // Clear data in correct order (respect foreign keys) - await postgresClient.query('DELETE FROM provider_mappings'); - await postgresClient.query('DELETE FROM symbols'); - await postgresClient.query('DELETE FROM exchanges'); - - // Reset sync status - await postgresClient.query( - 'UPDATE sync_status SET last_sync_at = NULL, last_sync_count = 0, sync_errors = NULL' - ); - - logger.info('PostgreSQL data cleared successfully'); -} - -async function syncEODExchanges(): Promise { - const mongoClient = getMongoDBClient(); - const exchanges = await mongoClient.find('eodExchanges', { active: true }); - const result: SyncResult = { processed: 0, created: 0, updated: 0, skipped: 0, errors: 0 }; - - for (const exchange of exchanges) { - try { - // Create provider exchange mapping for EOD - await createProviderExchangeMapping( - 'eod', // provider - exchange.Code, - exchange.Name, - exchange.CountryISO2, - exchange.Currency, - 0.95 // very high confidence for EOD data - ); - - result.processed++; - result.created++; // Count as created mapping - } catch (error) { - logger.error('Failed to process EOD exchange', { error, exchange }); - result.errors++; - } - } - - return result; -} - -async function syncIBExchanges(): Promise { - const mongoClient = getMongoDBClient(); - const exchanges = await mongoClient.find('ibExchanges', {}); - const result: SyncResult = { processed: 0, created: 0, updated: 0, skipped: 0, errors: 0 }; - - for (const exchange of exchanges) { - try { - // Create provider exchange mapping for IB - await createProviderExchangeMapping( - 'ib', // provider - exchange.exchange_id, - exchange.name, - exchange.country_code, - 'USD', // IB doesn't specify currency, default to USD - 0.85 // good confidence for IB data - ); - - result.processed++; - result.created++; // Count as created mapping - } catch (error) { - logger.error('Failed to process IB exchange', { error, exchange }); - result.errors++; - } - } - - return result; -} - -async function createProviderExchangeMapping( - provider: string, - providerExchangeCode: string, - providerExchangeName: string, - countryCode: string | null, - currency: string | null, - confidence: number -): Promise { - if (!providerExchangeCode) { - return; - } - - const postgresClient = getPostgreSQLClient(); - - // Check if mapping already exists - const existingMapping = await findProviderExchangeMapping(provider, providerExchangeCode); - if (existingMapping) { - // Don't override existing mappings to preserve manual work - return; - } - - // Find or create master exchange - const masterExchange = await findOrCreateMasterExchange( - providerExchangeCode, - providerExchangeName, - countryCode, - currency - ); - - // Create the provider exchange mapping - const query = ` - INSERT INTO provider_exchange_mappings - (provider, provider_exchange_code, provider_exchange_name, master_exchange_id, - country_code, currency, confidence, active, auto_mapped) - VALUES ($1, $2, $3, $4, $5, $6, $7, false, true) - ON CONFLICT (provider, provider_exchange_code) DO NOTHING - `; - - await postgresClient.query(query, [ - provider, - providerExchangeCode, - providerExchangeName, - masterExchange.id, - countryCode, - currency, - confidence, - ]); -} - -async function findOrCreateMasterExchange( - providerCode: string, - providerName: string, - countryCode: string | null, - currency: string | null -): Promise { - const postgresClient = getPostgreSQLClient(); - - // First, try to find exact match - let masterExchange = await findExchangeByCode(providerCode); - - if (masterExchange) { - return masterExchange; - } - - // Try to find by similar codes (basic mapping) - const basicMapping = getBasicExchangeMapping(providerCode); - if (basicMapping) { - masterExchange = await findExchangeByCode(basicMapping); - if (masterExchange) { - return masterExchange; - } - } - - // Create new master exchange (inactive by default) - const query = ` - INSERT INTO exchanges (code, name, country, currency, active) - VALUES ($1, $2, $3, $4, false) - ON CONFLICT (code) DO UPDATE SET - name = COALESCE(EXCLUDED.name, exchanges.name), - country = COALESCE(EXCLUDED.country, exchanges.country), - currency = COALESCE(EXCLUDED.currency, exchanges.currency) - RETURNING id, code, name, country, currency - `; - - const result = await postgresClient.query(query, [ - providerCode, - providerName || providerCode, - countryCode || 'US', - currency || 'USD', - ]); - - return result.rows[0]; -} - -function getBasicExchangeMapping(providerCode: string): string | null { - const mappings: Record = { - NYE: 'NYSE', - NAS: 'NASDAQ', - TO: 'TSX', - LN: 'LSE', - LON: 'LSE', - }; - - return mappings[providerCode.toUpperCase()] || null; -} - -async function findProviderExchangeMapping(provider: string, providerExchangeCode: string): Promise { - const postgresClient = getPostgreSQLClient(); - const query = 'SELECT * FROM provider_exchange_mappings WHERE provider = $1 AND provider_exchange_code = $2'; - const result = await postgresClient.query(query, [provider, providerExchangeCode]); - return result.rows[0] || null; -} - -async function findExchangeByCode(code: string): Promise { - const postgresClient = getPostgreSQLClient(); - const query = 'SELECT * FROM exchanges WHERE code = $1'; - const result = await postgresClient.query(query, [code]); - return result.rows[0] || null; -} - -async function updateSyncStatus(provider: string, dataType: string, count: number, postgresClient: any): Promise { - const query = ` - INSERT INTO sync_status (provider, data_type, last_sync_at, last_sync_count, sync_errors) - VALUES ($1, $2, NOW(), $3, NULL) - ON CONFLICT (provider, data_type) - DO UPDATE SET - last_sync_at = NOW(), - last_sync_count = EXCLUDED.last_sync_count, - sync_errors = NULL, - updated_at = NOW() - `; - - await postgresClient.query(query, [provider, dataType, count]); -} - -function mergeResults(target: SyncResult, source: SyncResult): void { - target.processed += source.processed; - target.created += source.created; - target.updated += source.updated; - target.skipped += source.skipped; - target.errors += source.errors; -} \ No newline at end of file +import { getLogger } from '@stock-bot/logger'; +import { getMongoDBClient, getPostgreSQLClient } from '../../../clients'; +import type { JobPayload, SyncResult } from '../../../types/job-payloads'; + +const logger = getLogger('enhanced-sync-all-exchanges'); + +export async function syncAllExchanges(payload: JobPayload): Promise { + const clearFirst = payload.clearFirst || true; + logger.info('Starting comprehensive exchange sync...', { clearFirst }); + + const result: SyncResult = { + processed: 0, + created: 0, + updated: 0, + skipped: 0, + errors: 0, + }; + + try { + const postgresClient = getPostgreSQLClient(); + + // Clear existing data if requested + if (clearFirst) { + await clearPostgreSQLData(postgresClient); + } + + // Start transaction for atomic operations + await postgresClient.query('BEGIN'); + + // 1. Sync from EOD exchanges (comprehensive global data) + const eodResult = await syncEODExchanges(); + mergeResults(result, eodResult); + + // 2. Sync from IB exchanges (detailed asset information) + const ibResult = await syncIBExchanges(); + mergeResults(result, ibResult); + + // 3. Update sync status + await updateSyncStatus('all', 'exchanges', result.processed, postgresClient); + + await postgresClient.query('COMMIT'); + + logger.info('Comprehensive exchange sync completed', result); + return result; + } catch (error) { + const postgresClient = getPostgreSQLClient(); + await postgresClient.query('ROLLBACK'); + logger.error('Comprehensive exchange sync failed', { error }); + throw error; + } +} + +async function clearPostgreSQLData(postgresClient: any): Promise { + logger.info('Clearing existing PostgreSQL data...'); + + // Clear data in correct order (respect foreign keys) + await postgresClient.query('DELETE FROM provider_mappings'); + await postgresClient.query('DELETE FROM symbols'); + await postgresClient.query('DELETE FROM exchanges'); + + // Reset sync status + await postgresClient.query( + 'UPDATE sync_status SET last_sync_at = NULL, last_sync_count = 0, sync_errors = NULL' + ); + + logger.info('PostgreSQL data cleared successfully'); +} + +async function syncEODExchanges(): Promise { + const mongoClient = getMongoDBClient(); + const exchanges = await mongoClient.find('eodExchanges', { active: true }); + const result: SyncResult = { processed: 0, created: 0, updated: 0, skipped: 0, errors: 0 }; + + for (const exchange of exchanges) { + try { + // Create provider exchange mapping for EOD + await createProviderExchangeMapping( + 'eod', // provider + exchange.Code, + exchange.Name, + exchange.CountryISO2, + exchange.Currency, + 0.95 // very high confidence for EOD data + ); + + result.processed++; + result.created++; // Count as created mapping + } catch (error) { + logger.error('Failed to process EOD exchange', { error, exchange }); + result.errors++; + } + } + + return result; +} + +async function syncIBExchanges(): Promise { + const mongoClient = getMongoDBClient(); + const exchanges = await mongoClient.find('ibExchanges', {}); + const result: SyncResult = { processed: 0, created: 0, updated: 0, skipped: 0, errors: 0 }; + + for (const exchange of exchanges) { + try { + // Create provider exchange mapping for IB + await createProviderExchangeMapping( + 'ib', // provider + exchange.exchange_id, + exchange.name, + exchange.country_code, + 'USD', // IB doesn't specify currency, default to USD + 0.85 // good confidence for IB data + ); + + result.processed++; + result.created++; // Count as created mapping + } catch (error) { + logger.error('Failed to process IB exchange', { error, exchange }); + result.errors++; + } + } + + return result; +} + +async function createProviderExchangeMapping( + provider: string, + providerExchangeCode: string, + providerExchangeName: string, + countryCode: string | null, + currency: string | null, + confidence: number +): Promise { + if (!providerExchangeCode) { + return; + } + + const postgresClient = getPostgreSQLClient(); + + // Check if mapping already exists + const existingMapping = await findProviderExchangeMapping(provider, providerExchangeCode); + if (existingMapping) { + // Don't override existing mappings to preserve manual work + return; + } + + // Find or create master exchange + const masterExchange = await findOrCreateMasterExchange( + providerExchangeCode, + providerExchangeName, + countryCode, + currency + ); + + // Create the provider exchange mapping + const query = ` + INSERT INTO provider_exchange_mappings + (provider, provider_exchange_code, provider_exchange_name, master_exchange_id, + country_code, currency, confidence, active, auto_mapped) + VALUES ($1, $2, $3, $4, $5, $6, $7, false, true) + ON CONFLICT (provider, provider_exchange_code) DO NOTHING + `; + + await postgresClient.query(query, [ + provider, + providerExchangeCode, + providerExchangeName, + masterExchange.id, + countryCode, + currency, + confidence, + ]); +} + +async function findOrCreateMasterExchange( + providerCode: string, + providerName: string, + countryCode: string | null, + currency: string | null +): Promise { + const postgresClient = getPostgreSQLClient(); + + // First, try to find exact match + let masterExchange = await findExchangeByCode(providerCode); + + if (masterExchange) { + return masterExchange; + } + + // Try to find by similar codes (basic mapping) + const basicMapping = getBasicExchangeMapping(providerCode); + if (basicMapping) { + masterExchange = await findExchangeByCode(basicMapping); + if (masterExchange) { + return masterExchange; + } + } + + // Create new master exchange (inactive by default) + const query = ` + INSERT INTO exchanges (code, name, country, currency, active) + VALUES ($1, $2, $3, $4, false) + ON CONFLICT (code) DO UPDATE SET + name = COALESCE(EXCLUDED.name, exchanges.name), + country = COALESCE(EXCLUDED.country, exchanges.country), + currency = COALESCE(EXCLUDED.currency, exchanges.currency) + RETURNING id, code, name, country, currency + `; + + const result = await postgresClient.query(query, [ + providerCode, + providerName || providerCode, + countryCode || 'US', + currency || 'USD', + ]); + + return result.rows[0]; +} + +function getBasicExchangeMapping(providerCode: string): string | null { + const mappings: Record = { + NYE: 'NYSE', + NAS: 'NASDAQ', + TO: 'TSX', + LN: 'LSE', + LON: 'LSE', + }; + + return mappings[providerCode.toUpperCase()] || null; +} + +async function findProviderExchangeMapping( + provider: string, + providerExchangeCode: string +): Promise { + const postgresClient = getPostgreSQLClient(); + const query = + 'SELECT * FROM provider_exchange_mappings WHERE provider = $1 AND provider_exchange_code = $2'; + const result = await postgresClient.query(query, [provider, providerExchangeCode]); + return result.rows[0] || null; +} + +async function findExchangeByCode(code: string): Promise { + const postgresClient = getPostgreSQLClient(); + const query = 'SELECT * FROM exchanges WHERE code = $1'; + const result = await postgresClient.query(query, [code]); + return result.rows[0] || null; +} + +async function updateSyncStatus( + provider: string, + dataType: string, + count: number, + postgresClient: any +): Promise { + const query = ` + INSERT INTO sync_status (provider, data_type, last_sync_at, last_sync_count, sync_errors) + VALUES ($1, $2, NOW(), $3, NULL) + ON CONFLICT (provider, data_type) + DO UPDATE SET + last_sync_at = NOW(), + last_sync_count = EXCLUDED.last_sync_count, + sync_errors = NULL, + updated_at = NOW() + `; + + await postgresClient.query(query, [provider, dataType, count]); +} + +function mergeResults(target: SyncResult, source: SyncResult): void { + target.processed += source.processed; + target.created += source.created; + target.updated += source.updated; + target.skipped += source.skipped; + target.errors += source.errors; +} diff --git a/apps/data-pipeline/src/handlers/exchanges/operations/sync-ib-exchanges.operations.ts b/apps/data-pipeline/src/handlers/exchanges/operations/sync-ib-exchanges.operations.ts index d8ba87d..d8da00c 100644 --- a/apps/data-pipeline/src/handlers/exchanges/operations/sync-ib-exchanges.operations.ts +++ b/apps/data-pipeline/src/handlers/exchanges/operations/sync-ib-exchanges.operations.ts @@ -1,206 +1,208 @@ -import { getLogger } from '@stock-bot/logger'; -import { getMongoDBClient } from '../../../clients'; -import type { JobPayload } from '../../../types/job-payloads'; -import type { MasterExchange } from '@stock-bot/mongodb'; - -const logger = getLogger('sync-ib-exchanges'); - -interface IBExchange { - id?: string; - _id?: any; - name?: string; - code?: string; - country_code?: string; - currency?: string; -} - -export async function syncIBExchanges(payload: JobPayload): Promise<{ syncedCount: number; totalExchanges: number }> { - logger.info('Syncing IB exchanges from database...'); - - try { - const mongoClient = getMongoDBClient(); - const db = mongoClient.getDatabase(); - - // Filter by country code US and CA - const ibExchanges = await db - .collection('ibExchanges') - .find({ - country_code: { $in: ['US', 'CA'] }, - }) - .toArray(); - - logger.info('Found IB exchanges in database', { count: ibExchanges.length }); - - let syncedCount = 0; - - for (const exchange of ibExchanges) { - try { - await createOrUpdateMasterExchange(exchange); - syncedCount++; - - logger.debug('Synced IB exchange', { - ibId: exchange.id, - country: exchange.country_code, - }); - } catch (error) { - logger.error('Failed to sync IB exchange', { exchange: exchange.id, error }); - } - } - - logger.info('IB exchange sync completed', { - syncedCount, - totalExchanges: ibExchanges.length, - }); - - return { syncedCount, totalExchanges: ibExchanges.length }; - } catch (error) { - logger.error('Failed to fetch IB exchanges from database', { error }); - return { syncedCount: 0, totalExchanges: 0 }; - } -} - -/** - * Create or update master exchange record 1:1 from IB exchange - */ -async function createOrUpdateMasterExchange(ibExchange: IBExchange): Promise { - const mongoClient = getMongoDBClient(); - const db = mongoClient.getDatabase(); - const collection = db.collection('masterExchanges'); - - const masterExchangeId = generateMasterExchangeId(ibExchange); - const now = new Date(); - - // Check if master exchange already exists - const existing = await collection.findOne({ masterExchangeId }); - - if (existing) { - // Update existing record - await collection.updateOne( - { masterExchangeId }, - { - $set: { - officialName: ibExchange.name || `Exchange ${ibExchange.id}`, - country: ibExchange.country_code || 'UNKNOWN', - currency: ibExchange.currency || 'USD', - timezone: inferTimezone(ibExchange), - updated_at: now, - }, - } - ); - - logger.debug('Updated existing master exchange', { masterExchangeId }); - } else { - // Create new master exchange - const masterExchange: MasterExchange = { - masterExchangeId, - shortName: masterExchangeId, // Set shortName to masterExchangeId on creation - officialName: ibExchange.name || `Exchange ${ibExchange.id}`, - country: ibExchange.country_code || 'UNKNOWN', - currency: ibExchange.currency || 'USD', - timezone: inferTimezone(ibExchange), - active: false, // Set active to false only on creation - - sourceMappings: { - ib: { - id: ibExchange.id || ibExchange._id?.toString() || 'unknown', - name: ibExchange.name || `Exchange ${ibExchange.id}`, - code: ibExchange.code || ibExchange.id || '', - aliases: generateAliases(ibExchange), - lastUpdated: now, - }, - }, - - confidence: 1.0, // High confidence for direct IB mapping - verified: true, // Mark as verified since it's direct from IB - - // DocumentBase fields - source: 'ib-exchange-sync', - created_at: now, - updated_at: now, - }; - - await collection.insertOne(masterExchange); - logger.debug('Created new master exchange', { masterExchangeId }); - } -} - -/** - * Generate master exchange ID from IB exchange - */ -function generateMasterExchangeId(ibExchange: IBExchange): string { - // Use code if available, otherwise use ID, otherwise generate from name - if (ibExchange.code) { - return ibExchange.code.toUpperCase().replace(/[^A-Z0-9]/g, ''); - } - - if (ibExchange.id) { - return ibExchange.id.toUpperCase().replace(/[^A-Z0-9]/g, ''); - } - - if (ibExchange.name) { - return ibExchange.name - .toUpperCase() - .split(' ') - .slice(0, 2) - .join('_') - .replace(/[^A-Z0-9_]/g, ''); - } - - return 'UNKNOWN_EXCHANGE'; -} - -/** - * Generate aliases for the exchange - */ -function generateAliases(ibExchange: IBExchange): string[] { - const aliases: string[] = []; - - if (ibExchange.name && ibExchange.name.includes(' ')) { - // Add abbreviated version - aliases.push( - ibExchange.name - .split(' ') - .map(w => w[0]) - .join('') - .toUpperCase() - ); - } - - if (ibExchange.code) { - aliases.push(ibExchange.code.toUpperCase()); - } - - return aliases; -} - -/** - * Infer timezone from exchange name/location - */ -function inferTimezone(ibExchange: IBExchange): string { - if (!ibExchange.name) { - return 'UTC'; - } - - const name = ibExchange.name.toUpperCase(); - - if (name.includes('NEW YORK') || name.includes('NYSE') || name.includes('NASDAQ')) { - return 'America/New_York'; - } - if (name.includes('LONDON')) { - return 'Europe/London'; - } - if (name.includes('TOKYO')) { - return 'Asia/Tokyo'; - } - if (name.includes('SHANGHAI')) { - return 'Asia/Shanghai'; - } - if (name.includes('TORONTO')) { - return 'America/Toronto'; - } - if (name.includes('FRANKFURT')) { - return 'Europe/Berlin'; - } - - return 'UTC'; // Default -} \ No newline at end of file +import { getLogger } from '@stock-bot/logger'; +import type { MasterExchange } from '@stock-bot/mongodb'; +import { getMongoDBClient } from '../../../clients'; +import type { JobPayload } from '../../../types/job-payloads'; + +const logger = getLogger('sync-ib-exchanges'); + +interface IBExchange { + id?: string; + _id?: any; + name?: string; + code?: string; + country_code?: string; + currency?: string; +} + +export async function syncIBExchanges( + payload: JobPayload +): Promise<{ syncedCount: number; totalExchanges: number }> { + logger.info('Syncing IB exchanges from database...'); + + try { + const mongoClient = getMongoDBClient(); + const db = mongoClient.getDatabase(); + + // Filter by country code US and CA + const ibExchanges = await db + .collection('ibExchanges') + .find({ + country_code: { $in: ['US', 'CA'] }, + }) + .toArray(); + + logger.info('Found IB exchanges in database', { count: ibExchanges.length }); + + let syncedCount = 0; + + for (const exchange of ibExchanges) { + try { + await createOrUpdateMasterExchange(exchange); + syncedCount++; + + logger.debug('Synced IB exchange', { + ibId: exchange.id, + country: exchange.country_code, + }); + } catch (error) { + logger.error('Failed to sync IB exchange', { exchange: exchange.id, error }); + } + } + + logger.info('IB exchange sync completed', { + syncedCount, + totalExchanges: ibExchanges.length, + }); + + return { syncedCount, totalExchanges: ibExchanges.length }; + } catch (error) { + logger.error('Failed to fetch IB exchanges from database', { error }); + return { syncedCount: 0, totalExchanges: 0 }; + } +} + +/** + * Create or update master exchange record 1:1 from IB exchange + */ +async function createOrUpdateMasterExchange(ibExchange: IBExchange): Promise { + const mongoClient = getMongoDBClient(); + const db = mongoClient.getDatabase(); + const collection = db.collection('masterExchanges'); + + const masterExchangeId = generateMasterExchangeId(ibExchange); + const now = new Date(); + + // Check if master exchange already exists + const existing = await collection.findOne({ masterExchangeId }); + + if (existing) { + // Update existing record + await collection.updateOne( + { masterExchangeId }, + { + $set: { + officialName: ibExchange.name || `Exchange ${ibExchange.id}`, + country: ibExchange.country_code || 'UNKNOWN', + currency: ibExchange.currency || 'USD', + timezone: inferTimezone(ibExchange), + updated_at: now, + }, + } + ); + + logger.debug('Updated existing master exchange', { masterExchangeId }); + } else { + // Create new master exchange + const masterExchange: MasterExchange = { + masterExchangeId, + shortName: masterExchangeId, // Set shortName to masterExchangeId on creation + officialName: ibExchange.name || `Exchange ${ibExchange.id}`, + country: ibExchange.country_code || 'UNKNOWN', + currency: ibExchange.currency || 'USD', + timezone: inferTimezone(ibExchange), + active: false, // Set active to false only on creation + + sourceMappings: { + ib: { + id: ibExchange.id || ibExchange._id?.toString() || 'unknown', + name: ibExchange.name || `Exchange ${ibExchange.id}`, + code: ibExchange.code || ibExchange.id || '', + aliases: generateAliases(ibExchange), + lastUpdated: now, + }, + }, + + confidence: 1.0, // High confidence for direct IB mapping + verified: true, // Mark as verified since it's direct from IB + + // DocumentBase fields + source: 'ib-exchange-sync', + created_at: now, + updated_at: now, + }; + + await collection.insertOne(masterExchange); + logger.debug('Created new master exchange', { masterExchangeId }); + } +} + +/** + * Generate master exchange ID from IB exchange + */ +function generateMasterExchangeId(ibExchange: IBExchange): string { + // Use code if available, otherwise use ID, otherwise generate from name + if (ibExchange.code) { + return ibExchange.code.toUpperCase().replace(/[^A-Z0-9]/g, ''); + } + + if (ibExchange.id) { + return ibExchange.id.toUpperCase().replace(/[^A-Z0-9]/g, ''); + } + + if (ibExchange.name) { + return ibExchange.name + .toUpperCase() + .split(' ') + .slice(0, 2) + .join('_') + .replace(/[^A-Z0-9_]/g, ''); + } + + return 'UNKNOWN_EXCHANGE'; +} + +/** + * Generate aliases for the exchange + */ +function generateAliases(ibExchange: IBExchange): string[] { + const aliases: string[] = []; + + if (ibExchange.name && ibExchange.name.includes(' ')) { + // Add abbreviated version + aliases.push( + ibExchange.name + .split(' ') + .map(w => w[0]) + .join('') + .toUpperCase() + ); + } + + if (ibExchange.code) { + aliases.push(ibExchange.code.toUpperCase()); + } + + return aliases; +} + +/** + * Infer timezone from exchange name/location + */ +function inferTimezone(ibExchange: IBExchange): string { + if (!ibExchange.name) { + return 'UTC'; + } + + const name = ibExchange.name.toUpperCase(); + + if (name.includes('NEW YORK') || name.includes('NYSE') || name.includes('NASDAQ')) { + return 'America/New_York'; + } + if (name.includes('LONDON')) { + return 'Europe/London'; + } + if (name.includes('TOKYO')) { + return 'Asia/Tokyo'; + } + if (name.includes('SHANGHAI')) { + return 'Asia/Shanghai'; + } + if (name.includes('TORONTO')) { + return 'America/Toronto'; + } + if (name.includes('FRANKFURT')) { + return 'Europe/Berlin'; + } + + return 'UTC'; // Default +} diff --git a/apps/data-pipeline/src/handlers/exchanges/operations/sync-qm-provider-mappings.operations.ts b/apps/data-pipeline/src/handlers/exchanges/operations/sync-qm-provider-mappings.operations.ts index 15e2fa0..ad7900e 100644 --- a/apps/data-pipeline/src/handlers/exchanges/operations/sync-qm-provider-mappings.operations.ts +++ b/apps/data-pipeline/src/handlers/exchanges/operations/sync-qm-provider-mappings.operations.ts @@ -1,203 +1,207 @@ -import { getLogger } from '@stock-bot/logger'; -import { getMongoDBClient, getPostgreSQLClient } from '../../../clients'; -import type { JobPayload, SyncResult } from '../../../types/job-payloads'; - -const logger = getLogger('enhanced-sync-qm-provider-mappings'); - -export async function syncQMProviderMappings(payload: JobPayload): Promise { - logger.info('Starting QM provider exchange mappings sync...'); - - const result: SyncResult = { - processed: 0, - created: 0, - updated: 0, - skipped: 0, - errors: 0, - }; - - try { - const mongoClient = getMongoDBClient(); - const postgresClient = getPostgreSQLClient(); - - // Start transaction - await postgresClient.query('BEGIN'); - - // Get unique exchange combinations from QM symbols - const db = mongoClient.getDatabase(); - const pipeline = [ - { - $group: { - _id: { - exchangeCode: '$exchangeCode', - exchange: '$exchange', - countryCode: '$countryCode', - }, - count: { $sum: 1 }, - sampleExchange: { $first: '$exchange' }, - }, - }, - { - $project: { - exchangeCode: '$_id.exchangeCode', - exchange: '$_id.exchange', - countryCode: '$_id.countryCode', - count: 1, - sampleExchange: 1, - }, - }, - ]; - - const qmExchanges = await db.collection('qmSymbols').aggregate(pipeline).toArray(); - logger.info(`Found ${qmExchanges.length} unique QM exchange combinations`); - - for (const exchange of qmExchanges) { - try { - // Create provider exchange mapping for QM - await createProviderExchangeMapping( - 'qm', // provider - exchange.exchangeCode, - exchange.sampleExchange || exchange.exchangeCode, - exchange.countryCode, - exchange.countryCode === 'CA' ? 'CAD' : 'USD', // Simple currency mapping - 0.8 // good confidence for QM data - ); - - result.processed++; - result.created++; - } catch (error) { - logger.error('Failed to process QM exchange mapping', { error, exchange }); - result.errors++; - } - } - - await postgresClient.query('COMMIT'); - - logger.info('QM provider exchange mappings sync completed', result); - return result; - } catch (error) { - const postgresClient = getPostgreSQLClient(); - await postgresClient.query('ROLLBACK'); - logger.error('QM provider exchange mappings sync failed', { error }); - throw error; - } -} - -async function createProviderExchangeMapping( - provider: string, - providerExchangeCode: string, - providerExchangeName: string, - countryCode: string | null, - currency: string | null, - confidence: number -): Promise { - if (!providerExchangeCode) { - return; - } - - const postgresClient = getPostgreSQLClient(); - - // Check if mapping already exists - const existingMapping = await findProviderExchangeMapping(provider, providerExchangeCode); - if (existingMapping) { - // Don't override existing mappings to preserve manual work - return; - } - - // Find or create master exchange - const masterExchange = await findOrCreateMasterExchange( - providerExchangeCode, - providerExchangeName, - countryCode, - currency - ); - - // Create the provider exchange mapping - const query = ` - INSERT INTO provider_exchange_mappings - (provider, provider_exchange_code, provider_exchange_name, master_exchange_id, - country_code, currency, confidence, active, auto_mapped) - VALUES ($1, $2, $3, $4, $5, $6, $7, false, true) - ON CONFLICT (provider, provider_exchange_code) DO NOTHING - `; - - await postgresClient.query(query, [ - provider, - providerExchangeCode, - providerExchangeName, - masterExchange.id, - countryCode, - currency, - confidence, - ]); -} - -async function findProviderExchangeMapping(provider: string, providerExchangeCode: string): Promise { - const postgresClient = getPostgreSQLClient(); - const query = 'SELECT * FROM provider_exchange_mappings WHERE provider = $1 AND provider_exchange_code = $2'; - const result = await postgresClient.query(query, [provider, providerExchangeCode]); - return result.rows[0] || null; -} - -async function findOrCreateMasterExchange( - providerCode: string, - providerName: string, - countryCode: string | null, - currency: string | null -): Promise { - const postgresClient = getPostgreSQLClient(); - - // First, try to find exact match - let masterExchange = await findExchangeByCode(providerCode); - - if (masterExchange) { - return masterExchange; - } - - // Try to find by similar codes (basic mapping) - const basicMapping = getBasicExchangeMapping(providerCode); - if (basicMapping) { - masterExchange = await findExchangeByCode(basicMapping); - if (masterExchange) { - return masterExchange; - } - } - - // Create new master exchange (inactive by default) - const query = ` - INSERT INTO exchanges (code, name, country, currency, active) - VALUES ($1, $2, $3, $4, false) - ON CONFLICT (code) DO UPDATE SET - name = COALESCE(EXCLUDED.name, exchanges.name), - country = COALESCE(EXCLUDED.country, exchanges.country), - currency = COALESCE(EXCLUDED.currency, exchanges.currency) - RETURNING id, code, name, country, currency - `; - - const result = await postgresClient.query(query, [ - providerCode, - providerName || providerCode, - countryCode || 'US', - currency || 'USD', - ]); - - return result.rows[0]; -} - -function getBasicExchangeMapping(providerCode: string): string | null { - const mappings: Record = { - NYE: 'NYSE', - NAS: 'NASDAQ', - TO: 'TSX', - LN: 'LSE', - LON: 'LSE', - }; - - return mappings[providerCode.toUpperCase()] || null; -} - -async function findExchangeByCode(code: string): Promise { - const postgresClient = getPostgreSQLClient(); - const query = 'SELECT * FROM exchanges WHERE code = $1'; - const result = await postgresClient.query(query, [code]); - return result.rows[0] || null; -} \ No newline at end of file +import { getLogger } from '@stock-bot/logger'; +import { getMongoDBClient, getPostgreSQLClient } from '../../../clients'; +import type { JobPayload, SyncResult } from '../../../types/job-payloads'; + +const logger = getLogger('enhanced-sync-qm-provider-mappings'); + +export async function syncQMProviderMappings(payload: JobPayload): Promise { + logger.info('Starting QM provider exchange mappings sync...'); + + const result: SyncResult = { + processed: 0, + created: 0, + updated: 0, + skipped: 0, + errors: 0, + }; + + try { + const mongoClient = getMongoDBClient(); + const postgresClient = getPostgreSQLClient(); + + // Start transaction + await postgresClient.query('BEGIN'); + + // Get unique exchange combinations from QM symbols + const db = mongoClient.getDatabase(); + const pipeline = [ + { + $group: { + _id: { + exchangeCode: '$exchangeCode', + exchange: '$exchange', + countryCode: '$countryCode', + }, + count: { $sum: 1 }, + sampleExchange: { $first: '$exchange' }, + }, + }, + { + $project: { + exchangeCode: '$_id.exchangeCode', + exchange: '$_id.exchange', + countryCode: '$_id.countryCode', + count: 1, + sampleExchange: 1, + }, + }, + ]; + + const qmExchanges = await db.collection('qmSymbols').aggregate(pipeline).toArray(); + logger.info(`Found ${qmExchanges.length} unique QM exchange combinations`); + + for (const exchange of qmExchanges) { + try { + // Create provider exchange mapping for QM + await createProviderExchangeMapping( + 'qm', // provider + exchange.exchangeCode, + exchange.sampleExchange || exchange.exchangeCode, + exchange.countryCode, + exchange.countryCode === 'CA' ? 'CAD' : 'USD', // Simple currency mapping + 0.8 // good confidence for QM data + ); + + result.processed++; + result.created++; + } catch (error) { + logger.error('Failed to process QM exchange mapping', { error, exchange }); + result.errors++; + } + } + + await postgresClient.query('COMMIT'); + + logger.info('QM provider exchange mappings sync completed', result); + return result; + } catch (error) { + const postgresClient = getPostgreSQLClient(); + await postgresClient.query('ROLLBACK'); + logger.error('QM provider exchange mappings sync failed', { error }); + throw error; + } +} + +async function createProviderExchangeMapping( + provider: string, + providerExchangeCode: string, + providerExchangeName: string, + countryCode: string | null, + currency: string | null, + confidence: number +): Promise { + if (!providerExchangeCode) { + return; + } + + const postgresClient = getPostgreSQLClient(); + + // Check if mapping already exists + const existingMapping = await findProviderExchangeMapping(provider, providerExchangeCode); + if (existingMapping) { + // Don't override existing mappings to preserve manual work + return; + } + + // Find or create master exchange + const masterExchange = await findOrCreateMasterExchange( + providerExchangeCode, + providerExchangeName, + countryCode, + currency + ); + + // Create the provider exchange mapping + const query = ` + INSERT INTO provider_exchange_mappings + (provider, provider_exchange_code, provider_exchange_name, master_exchange_id, + country_code, currency, confidence, active, auto_mapped) + VALUES ($1, $2, $3, $4, $5, $6, $7, false, true) + ON CONFLICT (provider, provider_exchange_code) DO NOTHING + `; + + await postgresClient.query(query, [ + provider, + providerExchangeCode, + providerExchangeName, + masterExchange.id, + countryCode, + currency, + confidence, + ]); +} + +async function findProviderExchangeMapping( + provider: string, + providerExchangeCode: string +): Promise { + const postgresClient = getPostgreSQLClient(); + const query = + 'SELECT * FROM provider_exchange_mappings WHERE provider = $1 AND provider_exchange_code = $2'; + const result = await postgresClient.query(query, [provider, providerExchangeCode]); + return result.rows[0] || null; +} + +async function findOrCreateMasterExchange( + providerCode: string, + providerName: string, + countryCode: string | null, + currency: string | null +): Promise { + const postgresClient = getPostgreSQLClient(); + + // First, try to find exact match + let masterExchange = await findExchangeByCode(providerCode); + + if (masterExchange) { + return masterExchange; + } + + // Try to find by similar codes (basic mapping) + const basicMapping = getBasicExchangeMapping(providerCode); + if (basicMapping) { + masterExchange = await findExchangeByCode(basicMapping); + if (masterExchange) { + return masterExchange; + } + } + + // Create new master exchange (inactive by default) + const query = ` + INSERT INTO exchanges (code, name, country, currency, active) + VALUES ($1, $2, $3, $4, false) + ON CONFLICT (code) DO UPDATE SET + name = COALESCE(EXCLUDED.name, exchanges.name), + country = COALESCE(EXCLUDED.country, exchanges.country), + currency = COALESCE(EXCLUDED.currency, exchanges.currency) + RETURNING id, code, name, country, currency + `; + + const result = await postgresClient.query(query, [ + providerCode, + providerName || providerCode, + countryCode || 'US', + currency || 'USD', + ]); + + return result.rows[0]; +} + +function getBasicExchangeMapping(providerCode: string): string | null { + const mappings: Record = { + NYE: 'NYSE', + NAS: 'NASDAQ', + TO: 'TSX', + LN: 'LSE', + LON: 'LSE', + }; + + return mappings[providerCode.toUpperCase()] || null; +} + +async function findExchangeByCode(code: string): Promise { + const postgresClient = getPostgreSQLClient(); + const query = 'SELECT * FROM exchanges WHERE code = $1'; + const result = await postgresClient.query(query, [code]); + return result.rows[0] || null; +} diff --git a/apps/data-pipeline/src/handlers/symbols/operations/index.ts b/apps/data-pipeline/src/handlers/symbols/operations/index.ts index 378fdd1..b4b431d 100644 --- a/apps/data-pipeline/src/handlers/symbols/operations/index.ts +++ b/apps/data-pipeline/src/handlers/symbols/operations/index.ts @@ -1,9 +1,9 @@ -import { syncQMSymbols } from './qm-symbols.operations'; -import { syncSymbolsFromProvider } from './sync-symbols-from-provider.operations'; -import { getSyncStatus } from './sync-status.operations'; - -export const symbolOperations = { - syncQMSymbols, - syncSymbolsFromProvider, - getSyncStatus, -}; \ No newline at end of file +import { syncQMSymbols } from './qm-symbols.operations'; +import { getSyncStatus } from './sync-status.operations'; +import { syncSymbolsFromProvider } from './sync-symbols-from-provider.operations'; + +export const symbolOperations = { + syncQMSymbols, + syncSymbolsFromProvider, + getSyncStatus, +}; diff --git a/apps/data-pipeline/src/handlers/symbols/operations/qm-symbols.operations.ts b/apps/data-pipeline/src/handlers/symbols/operations/qm-symbols.operations.ts index 0181983..5d40e0a 100644 --- a/apps/data-pipeline/src/handlers/symbols/operations/qm-symbols.operations.ts +++ b/apps/data-pipeline/src/handlers/symbols/operations/qm-symbols.operations.ts @@ -1,167 +1,183 @@ -import { getLogger } from '@stock-bot/logger'; -import { getMongoDBClient, getPostgreSQLClient } from '../../../clients'; -import type { JobPayload } from '../../../types/job-payloads'; - -const logger = getLogger('sync-qm-symbols'); - -export async function syncQMSymbols(payload: JobPayload): Promise<{ processed: number; created: number; updated: number }> { - logger.info('Starting QM symbols sync...'); - - try { - const mongoClient = getMongoDBClient(); - const postgresClient = getPostgreSQLClient(); - - // 1. Get all QM symbols from MongoDB - const qmSymbols = await mongoClient.find('qmSymbols', {}); - logger.info(`Found ${qmSymbols.length} QM symbols to process`); - - let created = 0; - let updated = 0; - - for (const symbol of qmSymbols) { - try { - // 2. Resolve exchange - const exchangeId = await resolveExchange(symbol.exchangeCode || symbol.exchange, postgresClient); - - if (!exchangeId) { - logger.warn('Unknown exchange, skipping symbol', { - symbol: symbol.symbol, - exchange: symbol.exchangeCode || symbol.exchange, - }); - continue; - } - - // 3. Check if symbol exists - const existingSymbol = await findSymbol(symbol.symbol, exchangeId, postgresClient); - - if (existingSymbol) { - // Update existing - await updateSymbol(existingSymbol.id, symbol, postgresClient); - await upsertProviderMapping(existingSymbol.id, 'qm', symbol, postgresClient); - updated++; - } else { - // Create new - const newSymbolId = await createSymbol(symbol, exchangeId, postgresClient); - await upsertProviderMapping(newSymbolId, 'qm', symbol, postgresClient); - created++; - } - } catch (error) { - logger.error('Failed to process symbol', { error, symbol: symbol.symbol }); - } - } - - // 4. Update sync status - await updateSyncStatus('qm', 'symbols', qmSymbols.length, postgresClient); - - const result = { processed: qmSymbols.length, created, updated }; - logger.info('QM symbols sync completed', result); - return result; - } catch (error) { - logger.error('QM symbols sync failed', { error }); - throw error; - } -} - -// Helper functions -async function resolveExchange(exchangeCode: string, postgresClient: any): Promise { - if (!exchangeCode) return null; - - // Simple mapping - expand this as needed - const exchangeMap: Record = { - NASDAQ: 'NASDAQ', - NYSE: 'NYSE', - TSX: 'TSX', - TSE: 'TSX', // TSE maps to TSX - LSE: 'LSE', - CME: 'CME', - }; - - const normalizedCode = exchangeMap[exchangeCode.toUpperCase()]; - if (!normalizedCode) { - return null; - } - - const query = 'SELECT id FROM exchanges WHERE code = $1'; - const result = await postgresClient.query(query, [normalizedCode]); - return result.rows[0]?.id || null; -} - -async function findSymbol(symbol: string, exchangeId: string, postgresClient: any): Promise { - const query = 'SELECT * FROM symbols WHERE symbol = $1 AND exchange_id = $2'; - const result = await postgresClient.query(query, [symbol, exchangeId]); - return result.rows[0] || null; -} - -async function createSymbol(qmSymbol: any, exchangeId: string, postgresClient: any): Promise { - const query = ` - INSERT INTO symbols (symbol, exchange_id, company_name, country, currency) - VALUES ($1, $2, $3, $4, $5) - RETURNING id - `; - - const result = await postgresClient.query(query, [ - qmSymbol.symbol, - exchangeId, - qmSymbol.companyName || qmSymbol.name, - qmSymbol.countryCode || 'US', - qmSymbol.currency || 'USD', - ]); - - return result.rows[0].id; -} - -async function updateSymbol(symbolId: string, qmSymbol: any, postgresClient: any): Promise { - const query = ` - UPDATE symbols - SET company_name = COALESCE($2, company_name), - country = COALESCE($3, country), - currency = COALESCE($4, currency), - updated_at = NOW() - WHERE id = $1 - `; - - await postgresClient.query(query, [ - symbolId, - qmSymbol.companyName || qmSymbol.name, - qmSymbol.countryCode, - qmSymbol.currency, - ]); -} - -async function upsertProviderMapping( - symbolId: string, - provider: string, - qmSymbol: any, - postgresClient: any -): Promise { - const query = ` - INSERT INTO provider_mappings - (symbol_id, provider, provider_symbol, provider_exchange, last_seen) - VALUES ($1, $2, $3, $4, NOW()) - ON CONFLICT (provider, provider_symbol) - DO UPDATE SET - symbol_id = EXCLUDED.symbol_id, - provider_exchange = EXCLUDED.provider_exchange, - last_seen = NOW() - `; - - await postgresClient.query(query, [ - symbolId, - provider, - qmSymbol.qmSearchCode || qmSymbol.symbol, - qmSymbol.exchangeCode || qmSymbol.exchange, - ]); -} - -async function updateSyncStatus(provider: string, dataType: string, count: number, postgresClient: any): Promise { - const query = ` - UPDATE sync_status - SET last_sync_at = NOW(), - last_sync_count = $3, - sync_errors = NULL, - updated_at = NOW() - WHERE provider = $1 AND data_type = $2 - `; - - await postgresClient.query(query, [provider, dataType, count]); -} \ No newline at end of file +import { getLogger } from '@stock-bot/logger'; +import { getMongoDBClient, getPostgreSQLClient } from '../../../clients'; +import type { JobPayload } from '../../../types/job-payloads'; + +const logger = getLogger('sync-qm-symbols'); + +export async function syncQMSymbols( + payload: JobPayload +): Promise<{ processed: number; created: number; updated: number }> { + logger.info('Starting QM symbols sync...'); + + try { + const mongoClient = getMongoDBClient(); + const postgresClient = getPostgreSQLClient(); + + // 1. Get all QM symbols from MongoDB + const qmSymbols = await mongoClient.find('qmSymbols', {}); + logger.info(`Found ${qmSymbols.length} QM symbols to process`); + + let created = 0; + let updated = 0; + + for (const symbol of qmSymbols) { + try { + // 2. Resolve exchange + const exchangeId = await resolveExchange( + symbol.exchangeCode || symbol.exchange, + postgresClient + ); + + if (!exchangeId) { + logger.warn('Unknown exchange, skipping symbol', { + symbol: symbol.symbol, + exchange: symbol.exchangeCode || symbol.exchange, + }); + continue; + } + + // 3. Check if symbol exists + const existingSymbol = await findSymbol(symbol.symbol, exchangeId, postgresClient); + + if (existingSymbol) { + // Update existing + await updateSymbol(existingSymbol.id, symbol, postgresClient); + await upsertProviderMapping(existingSymbol.id, 'qm', symbol, postgresClient); + updated++; + } else { + // Create new + const newSymbolId = await createSymbol(symbol, exchangeId, postgresClient); + await upsertProviderMapping(newSymbolId, 'qm', symbol, postgresClient); + created++; + } + } catch (error) { + logger.error('Failed to process symbol', { error, symbol: symbol.symbol }); + } + } + + // 4. Update sync status + await updateSyncStatus('qm', 'symbols', qmSymbols.length, postgresClient); + + const result = { processed: qmSymbols.length, created, updated }; + logger.info('QM symbols sync completed', result); + return result; + } catch (error) { + logger.error('QM symbols sync failed', { error }); + throw error; + } +} + +// Helper functions +async function resolveExchange(exchangeCode: string, postgresClient: any): Promise { + if (!exchangeCode) { + return null; + } + + // Simple mapping - expand this as needed + const exchangeMap: Record = { + NASDAQ: 'NASDAQ', + NYSE: 'NYSE', + TSX: 'TSX', + TSE: 'TSX', // TSE maps to TSX + LSE: 'LSE', + CME: 'CME', + }; + + const normalizedCode = exchangeMap[exchangeCode.toUpperCase()]; + if (!normalizedCode) { + return null; + } + + const query = 'SELECT id FROM exchanges WHERE code = $1'; + const result = await postgresClient.query(query, [normalizedCode]); + return result.rows[0]?.id || null; +} + +async function findSymbol(symbol: string, exchangeId: string, postgresClient: any): Promise { + const query = 'SELECT * FROM symbols WHERE symbol = $1 AND exchange_id = $2'; + const result = await postgresClient.query(query, [symbol, exchangeId]); + return result.rows[0] || null; +} + +async function createSymbol( + qmSymbol: any, + exchangeId: string, + postgresClient: any +): Promise { + const query = ` + INSERT INTO symbols (symbol, exchange_id, company_name, country, currency) + VALUES ($1, $2, $3, $4, $5) + RETURNING id + `; + + const result = await postgresClient.query(query, [ + qmSymbol.symbol, + exchangeId, + qmSymbol.companyName || qmSymbol.name, + qmSymbol.countryCode || 'US', + qmSymbol.currency || 'USD', + ]); + + return result.rows[0].id; +} + +async function updateSymbol(symbolId: string, qmSymbol: any, postgresClient: any): Promise { + const query = ` + UPDATE symbols + SET company_name = COALESCE($2, company_name), + country = COALESCE($3, country), + currency = COALESCE($4, currency), + updated_at = NOW() + WHERE id = $1 + `; + + await postgresClient.query(query, [ + symbolId, + qmSymbol.companyName || qmSymbol.name, + qmSymbol.countryCode, + qmSymbol.currency, + ]); +} + +async function upsertProviderMapping( + symbolId: string, + provider: string, + qmSymbol: any, + postgresClient: any +): Promise { + const query = ` + INSERT INTO provider_mappings + (symbol_id, provider, provider_symbol, provider_exchange, last_seen) + VALUES ($1, $2, $3, $4, NOW()) + ON CONFLICT (provider, provider_symbol) + DO UPDATE SET + symbol_id = EXCLUDED.symbol_id, + provider_exchange = EXCLUDED.provider_exchange, + last_seen = NOW() + `; + + await postgresClient.query(query, [ + symbolId, + provider, + qmSymbol.qmSearchCode || qmSymbol.symbol, + qmSymbol.exchangeCode || qmSymbol.exchange, + ]); +} + +async function updateSyncStatus( + provider: string, + dataType: string, + count: number, + postgresClient: any +): Promise { + const query = ` + UPDATE sync_status + SET last_sync_at = NOW(), + last_sync_count = $3, + sync_errors = NULL, + updated_at = NOW() + WHERE provider = $1 AND data_type = $2 + `; + + await postgresClient.query(query, [provider, dataType, count]); +} diff --git a/apps/data-pipeline/src/handlers/symbols/operations/sync-status.operations.ts b/apps/data-pipeline/src/handlers/symbols/operations/sync-status.operations.ts index ea83dbf..c4b70c5 100644 --- a/apps/data-pipeline/src/handlers/symbols/operations/sync-status.operations.ts +++ b/apps/data-pipeline/src/handlers/symbols/operations/sync-status.operations.ts @@ -1,21 +1,21 @@ -import { getLogger } from '@stock-bot/logger'; -import { getPostgreSQLClient } from '../../../clients'; -import type { JobPayload } from '../../../types/job-payloads'; - -const logger = getLogger('sync-status'); - -export async function getSyncStatus(payload: JobPayload): Promise[]> { - logger.info('Getting sync status...'); - - try { - const postgresClient = getPostgreSQLClient(); - const query = 'SELECT * FROM sync_status ORDER BY provider, data_type'; - const result = await postgresClient.query(query); - - logger.info(`Retrieved sync status for ${result.rows.length} entries`); - return result.rows; - } catch (error) { - logger.error('Failed to get sync status', { error }); - throw error; - } -} \ No newline at end of file +import { getLogger } from '@stock-bot/logger'; +import { getPostgreSQLClient } from '../../../clients'; +import type { JobPayload } from '../../../types/job-payloads'; + +const logger = getLogger('sync-status'); + +export async function getSyncStatus(payload: JobPayload): Promise[]> { + logger.info('Getting sync status...'); + + try { + const postgresClient = getPostgreSQLClient(); + const query = 'SELECT * FROM sync_status ORDER BY provider, data_type'; + const result = await postgresClient.query(query); + + logger.info(`Retrieved sync status for ${result.rows.length} entries`); + return result.rows; + } catch (error) { + logger.error('Failed to get sync status', { error }); + throw error; + } +} diff --git a/apps/data-pipeline/src/handlers/symbols/operations/sync-symbols-from-provider.operations.ts b/apps/data-pipeline/src/handlers/symbols/operations/sync-symbols-from-provider.operations.ts index 2e3308f..d965313 100644 --- a/apps/data-pipeline/src/handlers/symbols/operations/sync-symbols-from-provider.operations.ts +++ b/apps/data-pipeline/src/handlers/symbols/operations/sync-symbols-from-provider.operations.ts @@ -1,215 +1,231 @@ -import { getLogger } from '@stock-bot/logger'; -import { getMongoDBClient, getPostgreSQLClient } from '../../../clients'; -import type { JobPayload, SyncResult } from '../../../types/job-payloads'; - -const logger = getLogger('enhanced-sync-symbols-from-provider'); - -export async function syncSymbolsFromProvider(payload: JobPayload): Promise { - const provider = payload.provider; - const clearFirst = payload.clearFirst || false; - - if (!provider) { - throw new Error('Provider is required in payload'); - } - - logger.info(`Starting ${provider} symbols sync...`, { clearFirst }); - - const result: SyncResult = { - processed: 0, - created: 0, - updated: 0, - skipped: 0, - errors: 0, - }; - - try { - const mongoClient = getMongoDBClient(); - const postgresClient = getPostgreSQLClient(); - - // Clear existing data if requested (only symbols and mappings, keep exchanges) - if (clearFirst) { - await postgresClient.query('BEGIN'); - await postgresClient.query('DELETE FROM provider_mappings'); - await postgresClient.query('DELETE FROM symbols'); - await postgresClient.query('COMMIT'); - logger.info('Cleared existing symbols and mappings before sync'); - } - - // Start transaction - await postgresClient.query('BEGIN'); - - let symbols: Record[] = []; - - // Get symbols based on provider - const db = mongoClient.getDatabase(); - switch (provider.toLowerCase()) { - case 'qm': - symbols = await db.collection('qmSymbols').find({}).toArray(); - break; - case 'eod': - symbols = await db.collection('eodSymbols').find({}).toArray(); - break; - case 'ib': - symbols = await db.collection('ibSymbols').find({}).toArray(); - break; - default: - throw new Error(`Unsupported provider: ${provider}`); - } - - logger.info(`Found ${symbols.length} ${provider} symbols to process`); - result.processed = symbols.length; - - for (const symbol of symbols) { - try { - await processSingleSymbol(symbol, provider, result); - } catch (error) { - logger.error('Failed to process symbol', { - error, - symbol: symbol.symbol || symbol.code, - provider, - }); - result.errors++; - } - } - - // Update sync status - await updateSyncStatus(provider, 'symbols', result.processed, postgresClient); - - await postgresClient.query('COMMIT'); - - logger.info(`${provider} symbols sync completed`, result); - return result; - } catch (error) { - const postgresClient = getPostgreSQLClient(); - await postgresClient.query('ROLLBACK'); - logger.error(`${provider} symbols sync failed`, { error }); - throw error; - } -} - -async function processSingleSymbol(symbol: any, provider: string, result: SyncResult): Promise { - const symbolCode = symbol.symbol || symbol.code; - const exchangeCode = symbol.exchangeCode || symbol.exchange || symbol.exchange_id; - - if (!symbolCode || !exchangeCode) { - result.skipped++; - return; - } - - // Find active provider exchange mapping - const providerMapping = await findActiveProviderExchangeMapping(provider, exchangeCode); - - if (!providerMapping) { - result.skipped++; - return; - } - - // Check if symbol exists - const existingSymbol = await findSymbolByCodeAndExchange( - symbolCode, - providerMapping.master_exchange_id - ); - - if (existingSymbol) { - await updateSymbol(existingSymbol.id, symbol); - await upsertProviderMapping(existingSymbol.id, provider, symbol); - result.updated++; - } else { - const newSymbolId = await createSymbol(symbol, providerMapping.master_exchange_id); - await upsertProviderMapping(newSymbolId, provider, symbol); - result.created++; - } -} - -async function findActiveProviderExchangeMapping(provider: string, providerExchangeCode: string): Promise { - const postgresClient = getPostgreSQLClient(); - const query = ` - SELECT pem.*, e.code as master_exchange_code - FROM provider_exchange_mappings pem - JOIN exchanges e ON pem.master_exchange_id = e.id - WHERE pem.provider = $1 AND pem.provider_exchange_code = $2 AND pem.active = true - `; - const result = await postgresClient.query(query, [provider, providerExchangeCode]); - return result.rows[0] || null; -} - -async function findSymbolByCodeAndExchange(symbol: string, exchangeId: string): Promise { - const postgresClient = getPostgreSQLClient(); - const query = 'SELECT * FROM symbols WHERE symbol = $1 AND exchange_id = $2'; - const result = await postgresClient.query(query, [symbol, exchangeId]); - return result.rows[0] || null; -} - -async function createSymbol(symbol: any, exchangeId: string): Promise { - const postgresClient = getPostgreSQLClient(); - const query = ` - INSERT INTO symbols (symbol, exchange_id, company_name, country, currency) - VALUES ($1, $2, $3, $4, $5) - RETURNING id - `; - - const result = await postgresClient.query(query, [ - symbol.symbol || symbol.code, - exchangeId, - symbol.companyName || symbol.name || symbol.company_name, - symbol.countryCode || symbol.country_code || 'US', - symbol.currency || 'USD', - ]); - - return result.rows[0].id; -} - -async function updateSymbol(symbolId: string, symbol: any): Promise { - const postgresClient = getPostgreSQLClient(); - const query = ` - UPDATE symbols - SET company_name = COALESCE($2, company_name), - country = COALESCE($3, country), - currency = COALESCE($4, currency), - updated_at = NOW() - WHERE id = $1 - `; - - await postgresClient.query(query, [ - symbolId, - symbol.companyName || symbol.name || symbol.company_name, - symbol.countryCode || symbol.country_code, - symbol.currency, - ]); -} - -async function upsertProviderMapping(symbolId: string, provider: string, symbol: any): Promise { - const postgresClient = getPostgreSQLClient(); - const query = ` - INSERT INTO provider_mappings - (symbol_id, provider, provider_symbol, provider_exchange, last_seen) - VALUES ($1, $2, $3, $4, NOW()) - ON CONFLICT (provider, provider_symbol) - DO UPDATE SET - symbol_id = EXCLUDED.symbol_id, - provider_exchange = EXCLUDED.provider_exchange, - last_seen = NOW() - `; - - await postgresClient.query(query, [ - symbolId, - provider, - symbol.qmSearchCode || symbol.symbol || symbol.code, - symbol.exchangeCode || symbol.exchange || symbol.exchange_id, - ]); -} - -async function updateSyncStatus(provider: string, dataType: string, count: number, postgresClient: any): Promise { - const query = ` - INSERT INTO sync_status (provider, data_type, last_sync_at, last_sync_count, sync_errors) - VALUES ($1, $2, NOW(), $3, NULL) - ON CONFLICT (provider, data_type) - DO UPDATE SET - last_sync_at = NOW(), - last_sync_count = EXCLUDED.last_sync_count, - sync_errors = NULL, - updated_at = NOW() - `; - - await postgresClient.query(query, [provider, dataType, count]); -} \ No newline at end of file +import { getLogger } from '@stock-bot/logger'; +import { getMongoDBClient, getPostgreSQLClient } from '../../../clients'; +import type { JobPayload, SyncResult } from '../../../types/job-payloads'; + +const logger = getLogger('enhanced-sync-symbols-from-provider'); + +export async function syncSymbolsFromProvider(payload: JobPayload): Promise { + const provider = payload.provider; + const clearFirst = payload.clearFirst || false; + + if (!provider) { + throw new Error('Provider is required in payload'); + } + + logger.info(`Starting ${provider} symbols sync...`, { clearFirst }); + + const result: SyncResult = { + processed: 0, + created: 0, + updated: 0, + skipped: 0, + errors: 0, + }; + + try { + const mongoClient = getMongoDBClient(); + const postgresClient = getPostgreSQLClient(); + + // Clear existing data if requested (only symbols and mappings, keep exchanges) + if (clearFirst) { + await postgresClient.query('BEGIN'); + await postgresClient.query('DELETE FROM provider_mappings'); + await postgresClient.query('DELETE FROM symbols'); + await postgresClient.query('COMMIT'); + logger.info('Cleared existing symbols and mappings before sync'); + } + + // Start transaction + await postgresClient.query('BEGIN'); + + let symbols: Record[] = []; + + // Get symbols based on provider + const db = mongoClient.getDatabase(); + switch (provider.toLowerCase()) { + case 'qm': + symbols = await db.collection('qmSymbols').find({}).toArray(); + break; + case 'eod': + symbols = await db.collection('eodSymbols').find({}).toArray(); + break; + case 'ib': + symbols = await db.collection('ibSymbols').find({}).toArray(); + break; + default: + throw new Error(`Unsupported provider: ${provider}`); + } + + logger.info(`Found ${symbols.length} ${provider} symbols to process`); + result.processed = symbols.length; + + for (const symbol of symbols) { + try { + await processSingleSymbol(symbol, provider, result); + } catch (error) { + logger.error('Failed to process symbol', { + error, + symbol: symbol.symbol || symbol.code, + provider, + }); + result.errors++; + } + } + + // Update sync status + await updateSyncStatus(provider, 'symbols', result.processed, postgresClient); + + await postgresClient.query('COMMIT'); + + logger.info(`${provider} symbols sync completed`, result); + return result; + } catch (error) { + const postgresClient = getPostgreSQLClient(); + await postgresClient.query('ROLLBACK'); + logger.error(`${provider} symbols sync failed`, { error }); + throw error; + } +} + +async function processSingleSymbol( + symbol: any, + provider: string, + result: SyncResult +): Promise { + const symbolCode = symbol.symbol || symbol.code; + const exchangeCode = symbol.exchangeCode || symbol.exchange || symbol.exchange_id; + + if (!symbolCode || !exchangeCode) { + result.skipped++; + return; + } + + // Find active provider exchange mapping + const providerMapping = await findActiveProviderExchangeMapping(provider, exchangeCode); + + if (!providerMapping) { + result.skipped++; + return; + } + + // Check if symbol exists + const existingSymbol = await findSymbolByCodeAndExchange( + symbolCode, + providerMapping.master_exchange_id + ); + + if (existingSymbol) { + await updateSymbol(existingSymbol.id, symbol); + await upsertProviderMapping(existingSymbol.id, provider, symbol); + result.updated++; + } else { + const newSymbolId = await createSymbol(symbol, providerMapping.master_exchange_id); + await upsertProviderMapping(newSymbolId, provider, symbol); + result.created++; + } +} + +async function findActiveProviderExchangeMapping( + provider: string, + providerExchangeCode: string +): Promise { + const postgresClient = getPostgreSQLClient(); + const query = ` + SELECT pem.*, e.code as master_exchange_code + FROM provider_exchange_mappings pem + JOIN exchanges e ON pem.master_exchange_id = e.id + WHERE pem.provider = $1 AND pem.provider_exchange_code = $2 AND pem.active = true + `; + const result = await postgresClient.query(query, [provider, providerExchangeCode]); + return result.rows[0] || null; +} + +async function findSymbolByCodeAndExchange(symbol: string, exchangeId: string): Promise { + const postgresClient = getPostgreSQLClient(); + const query = 'SELECT * FROM symbols WHERE symbol = $1 AND exchange_id = $2'; + const result = await postgresClient.query(query, [symbol, exchangeId]); + return result.rows[0] || null; +} + +async function createSymbol(symbol: any, exchangeId: string): Promise { + const postgresClient = getPostgreSQLClient(); + const query = ` + INSERT INTO symbols (symbol, exchange_id, company_name, country, currency) + VALUES ($1, $2, $3, $4, $5) + RETURNING id + `; + + const result = await postgresClient.query(query, [ + symbol.symbol || symbol.code, + exchangeId, + symbol.companyName || symbol.name || symbol.company_name, + symbol.countryCode || symbol.country_code || 'US', + symbol.currency || 'USD', + ]); + + return result.rows[0].id; +} + +async function updateSymbol(symbolId: string, symbol: any): Promise { + const postgresClient = getPostgreSQLClient(); + const query = ` + UPDATE symbols + SET company_name = COALESCE($2, company_name), + country = COALESCE($3, country), + currency = COALESCE($4, currency), + updated_at = NOW() + WHERE id = $1 + `; + + await postgresClient.query(query, [ + symbolId, + symbol.companyName || symbol.name || symbol.company_name, + symbol.countryCode || symbol.country_code, + symbol.currency, + ]); +} + +async function upsertProviderMapping( + symbolId: string, + provider: string, + symbol: any +): Promise { + const postgresClient = getPostgreSQLClient(); + const query = ` + INSERT INTO provider_mappings + (symbol_id, provider, provider_symbol, provider_exchange, last_seen) + VALUES ($1, $2, $3, $4, NOW()) + ON CONFLICT (provider, provider_symbol) + DO UPDATE SET + symbol_id = EXCLUDED.symbol_id, + provider_exchange = EXCLUDED.provider_exchange, + last_seen = NOW() + `; + + await postgresClient.query(query, [ + symbolId, + provider, + symbol.qmSearchCode || symbol.symbol || symbol.code, + symbol.exchangeCode || symbol.exchange || symbol.exchange_id, + ]); +} + +async function updateSyncStatus( + provider: string, + dataType: string, + count: number, + postgresClient: any +): Promise { + const query = ` + INSERT INTO sync_status (provider, data_type, last_sync_at, last_sync_count, sync_errors) + VALUES ($1, $2, NOW(), $3, NULL) + ON CONFLICT (provider, data_type) + DO UPDATE SET + last_sync_at = NOW(), + last_sync_count = EXCLUDED.last_sync_count, + sync_errors = NULL, + updated_at = NOW() + `; + + await postgresClient.query(query, [provider, dataType, count]); +} diff --git a/apps/data-pipeline/src/handlers/symbols/symbols.handler.ts b/apps/data-pipeline/src/handlers/symbols/symbols.handler.ts index 6fdd17f..9013e06 100644 --- a/apps/data-pipeline/src/handlers/symbols/symbols.handler.ts +++ b/apps/data-pipeline/src/handlers/symbols/symbols.handler.ts @@ -1,41 +1,41 @@ -import { getLogger } from '@stock-bot/logger'; -import { handlerRegistry, type HandlerConfig, type ScheduledJobConfig } from '@stock-bot/queue'; -import { symbolOperations } from './operations'; - -const logger = getLogger('symbols-handler'); - -const HANDLER_NAME = 'symbols'; - -const symbolsHandlerConfig: HandlerConfig = { - concurrency: 1, - maxAttempts: 3, - scheduledJobs: [ - { - operation: 'sync-qm-symbols', - cronPattern: '0 2 * * *', // Daily at 2 AM - payload: {}, - priority: 5, - immediately: false, - } as ScheduledJobConfig, - { - operation: 'sync-symbols-qm', - cronPattern: '0 4 * * *', // Daily at 4 AM - payload: { provider: 'qm', clearFirst: false }, - priority: 5, - immediately: false, - } as ScheduledJobConfig, - ], - operations: { - 'sync-qm-symbols': symbolOperations.syncQMSymbols, - 'sync-symbols-qm': symbolOperations.syncSymbolsFromProvider, - 'sync-symbols-eod': symbolOperations.syncSymbolsFromProvider, - 'sync-symbols-ib': symbolOperations.syncSymbolsFromProvider, - 'sync-status': symbolOperations.getSyncStatus, - }, -}; - -export function initializeSymbolsHandler(): void { - logger.info('Registering symbols handler...'); - handlerRegistry.registerHandler(HANDLER_NAME, symbolsHandlerConfig); - logger.info('Symbols handler registered successfully'); -} \ No newline at end of file +import { getLogger } from '@stock-bot/logger'; +import { handlerRegistry, type HandlerConfig, type ScheduledJobConfig } from '@stock-bot/queue'; +import { symbolOperations } from './operations'; + +const logger = getLogger('symbols-handler'); + +const HANDLER_NAME = 'symbols'; + +const symbolsHandlerConfig: HandlerConfig = { + concurrency: 1, + maxAttempts: 3, + scheduledJobs: [ + { + operation: 'sync-qm-symbols', + cronPattern: '0 2 * * *', // Daily at 2 AM + payload: {}, + priority: 5, + immediately: false, + } as ScheduledJobConfig, + { + operation: 'sync-symbols-qm', + cronPattern: '0 4 * * *', // Daily at 4 AM + payload: { provider: 'qm', clearFirst: false }, + priority: 5, + immediately: false, + } as ScheduledJobConfig, + ], + operations: { + 'sync-qm-symbols': symbolOperations.syncQMSymbols, + 'sync-symbols-qm': symbolOperations.syncSymbolsFromProvider, + 'sync-symbols-eod': symbolOperations.syncSymbolsFromProvider, + 'sync-symbols-ib': symbolOperations.syncSymbolsFromProvider, + 'sync-status': symbolOperations.getSyncStatus, + }, +}; + +export function initializeSymbolsHandler(): void { + logger.info('Registering symbols handler...'); + handlerRegistry.registerHandler(HANDLER_NAME, symbolsHandlerConfig); + logger.info('Symbols handler registered successfully'); +} diff --git a/apps/data-pipeline/src/index.ts b/apps/data-pipeline/src/index.ts index 4ad4f33..d5f487d 100644 --- a/apps/data-pipeline/src/index.ts +++ b/apps/data-pipeline/src/index.ts @@ -1,16 +1,16 @@ // Framework imports -import { initializeServiceConfig } from '@stock-bot/config'; import { Hono } from 'hono'; import { cors } from 'hono/cors'; +import { initializeServiceConfig } from '@stock-bot/config'; // Library imports import { getLogger, setLoggerConfig, shutdownLoggers } from '@stock-bot/logger'; import { MongoDBClient } from '@stock-bot/mongodb'; import { PostgreSQLClient } from '@stock-bot/postgres'; import { QueueManager, type QueueManagerConfig } from '@stock-bot/queue'; import { Shutdown } from '@stock-bot/shutdown'; +import { setMongoDBClient, setPostgreSQLClient } from './clients'; // Local imports import { enhancedSyncRoutes, healthRoutes, statsRoutes, syncRoutes } from './routes'; -import { setMongoDBClient, setPostgreSQLClient } from './clients'; const config = initializeServiceConfig(); console.log('Data Sync Service Configuration:', JSON.stringify(config, null, 2)); @@ -66,17 +66,20 @@ async function initializeServices() { // Initialize MongoDB client logger.debug('Connecting to MongoDB...'); const mongoConfig = databaseConfig.mongodb; - mongoClient = new MongoDBClient({ - uri: mongoConfig.uri, - database: mongoConfig.database, - host: mongoConfig.host || 'localhost', - port: mongoConfig.port || 27017, - timeouts: { - connectTimeout: 30000, - socketTimeout: 30000, - serverSelectionTimeout: 5000, + mongoClient = new MongoDBClient( + { + uri: mongoConfig.uri, + database: mongoConfig.database, + host: mongoConfig.host || 'localhost', + port: mongoConfig.port || 27017, + timeouts: { + connectTimeout: 30000, + socketTimeout: 30000, + serverSelectionTimeout: 5000, + }, }, - }, logger); + logger + ); await mongoClient.connect(); setMongoDBClient(mongoClient); logger.info('MongoDB connected'); @@ -84,18 +87,21 @@ async function initializeServices() { // Initialize PostgreSQL client logger.debug('Connecting to PostgreSQL...'); const pgConfig = databaseConfig.postgres; - postgresClient = new PostgreSQLClient({ - host: pgConfig.host, - port: pgConfig.port, - database: pgConfig.database, - username: pgConfig.user, - password: pgConfig.password, - poolSettings: { - min: 2, - max: pgConfig.poolSize || 10, - idleTimeoutMillis: pgConfig.idleTimeout || 30000, + postgresClient = new PostgreSQLClient( + { + host: pgConfig.host, + port: pgConfig.port, + database: pgConfig.database, + username: pgConfig.user, + password: pgConfig.password, + poolSettings: { + min: 2, + max: pgConfig.poolSize || 10, + idleTimeoutMillis: pgConfig.idleTimeout || 30000, + }, }, - }, logger); + logger + ); await postgresClient.connect(); setPostgreSQLClient(postgresClient); logger.info('PostgreSQL connected'); @@ -124,7 +130,7 @@ async function initializeServices() { enableDLQ: true, }, enableScheduledJobs: true, - delayWorkerStart: true, // Prevent workers from starting until all singletons are ready + delayWorkerStart: true, // Prevent workers from starting until all singletons are ready }; queueManager = QueueManager.getOrInitialize(queueManagerConfig); @@ -134,10 +140,10 @@ async function initializeServices() { logger.debug('Initializing sync handlers...'); const { initializeExchangesHandler } = await import('./handlers/exchanges/exchanges.handler'); const { initializeSymbolsHandler } = await import('./handlers/symbols/symbols.handler'); - + initializeExchangesHandler(); initializeSymbolsHandler(); - + logger.info('Sync handlers initialized'); // Create scheduled jobs from registered handlers @@ -271,4 +277,4 @@ startServer().catch(error => { process.exit(1); }); -logger.info('Data sync service startup initiated'); \ No newline at end of file +logger.info('Data sync service startup initiated'); diff --git a/apps/data-pipeline/src/routes/enhanced-sync.routes.ts b/apps/data-pipeline/src/routes/enhanced-sync.routes.ts index ba17805..474e550 100644 --- a/apps/data-pipeline/src/routes/enhanced-sync.routes.ts +++ b/apps/data-pipeline/src/routes/enhanced-sync.routes.ts @@ -11,13 +11,13 @@ enhancedSync.post('/exchanges/all', async c => { const clearFirst = c.req.query('clear') === 'true'; const queueManager = QueueManager.getInstance(); const exchangesQueue = queueManager.getQueue('exchanges'); - + const job = await exchangesQueue.addJob('sync-all-exchanges', { handler: 'exchanges', operation: 'sync-all-exchanges', payload: { clearFirst }, }); - + return c.json({ success: true, jobId: job.id, message: 'Enhanced exchange sync job queued' }); } catch (error) { logger.error('Failed to queue enhanced exchange sync job', { error }); @@ -32,14 +32,18 @@ enhancedSync.post('/provider-mappings/qm', async c => { try { const queueManager = QueueManager.getInstance(); const exchangesQueue = queueManager.getQueue('exchanges'); - + const job = await exchangesQueue.addJob('sync-qm-provider-mappings', { handler: 'exchanges', operation: 'sync-qm-provider-mappings', payload: {}, }); - - return c.json({ success: true, jobId: job.id, message: 'QM provider mappings sync job queued' }); + + return c.json({ + success: true, + jobId: job.id, + message: 'QM provider mappings sync job queued', + }); } catch (error) { logger.error('Failed to queue QM provider mappings sync job', { error }); return c.json( @@ -55,13 +59,13 @@ enhancedSync.post('/symbols/:provider', async c => { const clearFirst = c.req.query('clear') === 'true'; const queueManager = QueueManager.getInstance(); const symbolsQueue = queueManager.getQueue('symbols'); - + const job = await symbolsQueue.addJob(`sync-symbols-${provider}`, { handler: 'symbols', operation: `sync-symbols-${provider}`, payload: { provider, clearFirst }, }); - + return c.json({ success: true, jobId: job.id, message: `${provider} symbols sync job queued` }); } catch (error) { logger.error('Failed to queue enhanced symbol sync job', { error }); @@ -77,13 +81,13 @@ enhancedSync.get('/status/enhanced', async c => { try { const queueManager = QueueManager.getInstance(); const exchangesQueue = queueManager.getQueue('exchanges'); - + const job = await exchangesQueue.addJob('enhanced-sync-status', { handler: 'exchanges', operation: 'enhanced-sync-status', payload: {}, }); - + // Wait for job to complete and return result const result = await job.waitUntilFinished(); return c.json(result); @@ -93,4 +97,4 @@ enhancedSync.get('/status/enhanced', async c => { } }); -export { enhancedSync as enhancedSyncRoutes }; \ No newline at end of file +export { enhancedSync as enhancedSyncRoutes }; diff --git a/apps/data-pipeline/src/routes/index.ts b/apps/data-pipeline/src/routes/index.ts index b106768..143fac2 100644 --- a/apps/data-pipeline/src/routes/index.ts +++ b/apps/data-pipeline/src/routes/index.ts @@ -2,4 +2,4 @@ export { healthRoutes } from './health.routes'; export { syncRoutes } from './sync.routes'; export { enhancedSyncRoutes } from './enhanced-sync.routes'; -export { statsRoutes } from './stats.routes'; \ No newline at end of file +export { statsRoutes } from './stats.routes'; diff --git a/apps/data-pipeline/src/routes/stats.routes.ts b/apps/data-pipeline/src/routes/stats.routes.ts index 8112c9c..9c8c488 100644 --- a/apps/data-pipeline/src/routes/stats.routes.ts +++ b/apps/data-pipeline/src/routes/stats.routes.ts @@ -10,13 +10,13 @@ stats.get('/exchanges', async c => { try { const queueManager = QueueManager.getInstance(); const exchangesQueue = queueManager.getQueue('exchanges'); - + const job = await exchangesQueue.addJob('get-exchange-stats', { handler: 'exchanges', operation: 'get-exchange-stats', payload: {}, }); - + // Wait for job to complete and return result const result = await job.waitUntilFinished(); return c.json(result); @@ -30,13 +30,13 @@ stats.get('/provider-mappings', async c => { try { const queueManager = QueueManager.getInstance(); const exchangesQueue = queueManager.getQueue('exchanges'); - + const job = await exchangesQueue.addJob('get-provider-mapping-stats', { handler: 'exchanges', operation: 'get-provider-mapping-stats', payload: {}, }); - + // Wait for job to complete and return result const result = await job.waitUntilFinished(); return c.json(result); @@ -46,4 +46,4 @@ stats.get('/provider-mappings', async c => { } }); -export { stats as statsRoutes }; \ No newline at end of file +export { stats as statsRoutes }; diff --git a/apps/data-pipeline/src/routes/sync.routes.ts b/apps/data-pipeline/src/routes/sync.routes.ts index 487e31d..8bf40b7 100644 --- a/apps/data-pipeline/src/routes/sync.routes.ts +++ b/apps/data-pipeline/src/routes/sync.routes.ts @@ -10,13 +10,13 @@ sync.post('/symbols', async c => { try { const queueManager = QueueManager.getInstance(); const symbolsQueue = queueManager.getQueue('symbols'); - + const job = await symbolsQueue.addJob('sync-qm-symbols', { handler: 'symbols', operation: 'sync-qm-symbols', payload: {}, }); - + return c.json({ success: true, jobId: job.id, message: 'QM symbols sync job queued' }); } catch (error) { logger.error('Failed to queue symbol sync job', { error }); @@ -31,13 +31,13 @@ sync.post('/exchanges', async c => { try { const queueManager = QueueManager.getInstance(); const exchangesQueue = queueManager.getQueue('exchanges'); - + const job = await exchangesQueue.addJob('sync-qm-exchanges', { handler: 'exchanges', operation: 'sync-qm-exchanges', payload: {}, }); - + return c.json({ success: true, jobId: job.id, message: 'QM exchanges sync job queued' }); } catch (error) { logger.error('Failed to queue exchange sync job', { error }); @@ -53,13 +53,13 @@ sync.get('/status', async c => { try { const queueManager = QueueManager.getInstance(); const symbolsQueue = queueManager.getQueue('symbols'); - + const job = await symbolsQueue.addJob('sync-status', { handler: 'symbols', operation: 'sync-status', payload: {}, }); - + // Wait for job to complete and return result const result = await job.waitUntilFinished(); return c.json(result); @@ -74,13 +74,13 @@ sync.post('/clear', async c => { try { const queueManager = QueueManager.getInstance(); const exchangesQueue = queueManager.getQueue('exchanges'); - + const job = await exchangesQueue.addJob('clear-postgresql-data', { handler: 'exchanges', operation: 'clear-postgresql-data', payload: {}, }); - + // Wait for job to complete and return result const result = await job.waitUntilFinished(); return c.json({ success: true, result }); @@ -93,4 +93,4 @@ sync.post('/clear', async c => { } }); -export { sync as syncRoutes }; \ No newline at end of file +export { sync as syncRoutes }; diff --git a/apps/data-pipeline/src/types/job-payloads.ts b/apps/data-pipeline/src/types/job-payloads.ts index 6d53852..6c5f9de 100644 --- a/apps/data-pipeline/src/types/job-payloads.ts +++ b/apps/data-pipeline/src/types/job-payloads.ts @@ -1,27 +1,27 @@ -export interface JobPayload { - [key: string]: any; -} - -export interface SyncResult { - processed: number; - created: number; - updated: number; - skipped: number; - errors: number; -} - -export interface SyncStatus { - provider: string; - dataType: string; - lastSyncAt?: Date; - lastSyncCount: number; - syncErrors?: string; -} - -export interface ExchangeMapping { - id: string; - code: string; - name: string; - country: string; - currency: string; -} \ No newline at end of file +export interface JobPayload { + [key: string]: any; +} + +export interface SyncResult { + processed: number; + created: number; + updated: number; + skipped: number; + errors: number; +} + +export interface SyncStatus { + provider: string; + dataType: string; + lastSyncAt?: Date; + lastSyncCount: number; + syncErrors?: string; +} + +export interface ExchangeMapping { + id: string; + code: string; + name: string; + country: string; + currency: string; +} diff --git a/apps/web-api/config/default.json b/apps/web-api/config/default.json index dacdb89..4770e43 100644 --- a/apps/web-api/config/default.json +++ b/apps/web-api/config/default.json @@ -1,15 +1,15 @@ -{ - "service": { - "name": "web-api", - "port": 4000, - "host": "0.0.0.0", - "healthCheckPath": "/health", - "metricsPath": "/metrics", - "shutdownTimeout": 30000, - "cors": { - "enabled": true, - "origin": ["http://localhost:4200", "http://localhost:3000", "http://localhost:3002"], - "credentials": true - } - } -} \ No newline at end of file +{ + "service": { + "name": "web-api", + "port": 4000, + "host": "0.0.0.0", + "healthCheckPath": "/health", + "metricsPath": "/metrics", + "shutdownTimeout": 30000, + "cors": { + "enabled": true, + "origin": ["http://localhost:4200", "http://localhost:3000", "http://localhost:3002"], + "credentials": true + } + } +} diff --git a/apps/web-api/src/clients.ts b/apps/web-api/src/clients.ts index 4bc8217..8cd54e2 100644 --- a/apps/web-api/src/clients.ts +++ b/apps/web-api/src/clients.ts @@ -1,27 +1,27 @@ -import { PostgreSQLClient } from '@stock-bot/postgres'; -import { MongoDBClient } from '@stock-bot/mongodb'; - -let postgresClient: PostgreSQLClient | null = null; -let mongodbClient: MongoDBClient | null = null; - -export function setPostgreSQLClient(client: PostgreSQLClient): void { - postgresClient = client; -} - -export function getPostgreSQLClient(): PostgreSQLClient { - if (!postgresClient) { - throw new Error('PostgreSQL client not initialized. Call setPostgreSQLClient first.'); - } - return postgresClient; -} - -export function setMongoDBClient(client: MongoDBClient): void { - mongodbClient = client; -} - -export function getMongoDBClient(): MongoDBClient { - if (!mongodbClient) { - throw new Error('MongoDB client not initialized. Call setMongoDBClient first.'); - } - return mongodbClient; -} \ No newline at end of file +import { MongoDBClient } from '@stock-bot/mongodb'; +import { PostgreSQLClient } from '@stock-bot/postgres'; + +let postgresClient: PostgreSQLClient | null = null; +let mongodbClient: MongoDBClient | null = null; + +export function setPostgreSQLClient(client: PostgreSQLClient): void { + postgresClient = client; +} + +export function getPostgreSQLClient(): PostgreSQLClient { + if (!postgresClient) { + throw new Error('PostgreSQL client not initialized. Call setPostgreSQLClient first.'); + } + return postgresClient; +} + +export function setMongoDBClient(client: MongoDBClient): void { + mongodbClient = client; +} + +export function getMongoDBClient(): MongoDBClient { + if (!mongodbClient) { + throw new Error('MongoDB client not initialized. Call setMongoDBClient first.'); + } + return mongodbClient; +} diff --git a/apps/web-api/src/index.ts b/apps/web-api/src/index.ts index 939b743..4d247a4 100644 --- a/apps/web-api/src/index.ts +++ b/apps/web-api/src/index.ts @@ -77,17 +77,20 @@ async function initializeServices() { // Initialize MongoDB client logger.debug('Connecting to MongoDB...'); const mongoConfig = databaseConfig.mongodb; - mongoClient = new MongoDBClient({ - uri: mongoConfig.uri, - database: mongoConfig.database, - host: mongoConfig.host, - port: mongoConfig.port, - timeouts: { - connectTimeout: 30000, - socketTimeout: 30000, - serverSelectionTimeout: 5000, + mongoClient = new MongoDBClient( + { + uri: mongoConfig.uri, + database: mongoConfig.database, + host: mongoConfig.host, + port: mongoConfig.port, + timeouts: { + connectTimeout: 30000, + socketTimeout: 30000, + serverSelectionTimeout: 5000, + }, }, - }, logger); + logger + ); await mongoClient.connect(); setMongoDBClient(mongoClient); logger.info('MongoDB connected'); @@ -95,18 +98,21 @@ async function initializeServices() { // Initialize PostgreSQL client logger.debug('Connecting to PostgreSQL...'); const pgConfig = databaseConfig.postgres; - postgresClient = new PostgreSQLClient({ - host: pgConfig.host, - port: pgConfig.port, - database: pgConfig.database, - username: pgConfig.user, - password: pgConfig.password, - poolSettings: { - min: 2, - max: pgConfig.poolSize || 10, - idleTimeoutMillis: pgConfig.idleTimeout || 30000, + postgresClient = new PostgreSQLClient( + { + host: pgConfig.host, + port: pgConfig.port, + database: pgConfig.database, + username: pgConfig.user, + password: pgConfig.password, + poolSettings: { + min: 2, + max: pgConfig.poolSize || 10, + idleTimeoutMillis: pgConfig.idleTimeout || 30000, + }, }, - }, logger); + logger + ); await postgresClient.connect(); setPostgreSQLClient(postgresClient); logger.info('PostgreSQL connected'); diff --git a/apps/web-api/src/routes/exchange.routes.ts b/apps/web-api/src/routes/exchange.routes.ts index 3e189fe..666b411 100644 --- a/apps/web-api/src/routes/exchange.routes.ts +++ b/apps/web-api/src/routes/exchange.routes.ts @@ -4,13 +4,13 @@ import { Hono } from 'hono'; import { getLogger } from '@stock-bot/logger'; import { exchangeService } from '../services/exchange.service'; +import { createSuccessResponse, handleError } from '../utils/error-handler'; import { validateCreateExchange, - validateUpdateExchange, validateCreateProviderMapping, + validateUpdateExchange, validateUpdateProviderMapping, } from '../utils/validation'; -import { handleError, createSuccessResponse } from '../utils/error-handler'; const logger = getLogger('exchange-routes'); export const exchangeRoutes = new Hono(); @@ -32,19 +32,19 @@ exchangeRoutes.get('/', async c => { exchangeRoutes.get('/:id', async c => { const exchangeId = c.req.param('id'); logger.debug('Getting exchange by ID', { exchangeId }); - + try { const result = await exchangeService.getExchangeById(exchangeId); - + if (!result) { logger.warn('Exchange not found', { exchangeId }); return c.json(createSuccessResponse(null, 'Exchange not found'), 404); } - logger.info('Successfully retrieved exchange details', { - exchangeId, + logger.info('Successfully retrieved exchange details', { + exchangeId, exchangeCode: result.exchange.code, - mappingCount: result.provider_mappings.length + mappingCount: result.provider_mappings.length, }); return c.json(createSuccessResponse(result)); } catch (error) { @@ -56,25 +56,22 @@ exchangeRoutes.get('/:id', async c => { // Create new exchange exchangeRoutes.post('/', async c => { logger.debug('Creating new exchange'); - + try { const body = await c.req.json(); logger.debug('Received exchange creation request', { requestBody: body }); - + const validatedData = validateCreateExchange(body); logger.debug('Exchange data validated successfully', { validatedData }); - + const exchange = await exchangeService.createExchange(validatedData); - logger.info('Exchange created successfully', { + logger.info('Exchange created successfully', { exchangeId: exchange.id, code: exchange.code, - name: exchange.name + name: exchange.name, }); - - return c.json( - createSuccessResponse(exchange, 'Exchange created successfully'), - 201 - ); + + return c.json(createSuccessResponse(exchange, 'Exchange created successfully'), 201); } catch (error) { logger.error('Failed to create exchange', { error }); return handleError(c, error, 'to create exchange'); @@ -85,32 +82,32 @@ exchangeRoutes.post('/', async c => { exchangeRoutes.patch('/:id', async c => { const exchangeId = c.req.param('id'); logger.debug('Updating exchange', { exchangeId }); - + try { const body = await c.req.json(); logger.debug('Received exchange update request', { exchangeId, updates: body }); - + const validatedUpdates = validateUpdateExchange(body); logger.debug('Exchange update data validated', { exchangeId, validatedUpdates }); - + const exchange = await exchangeService.updateExchange(exchangeId, validatedUpdates); - + if (!exchange) { logger.warn('Exchange not found for update', { exchangeId }); return c.json(createSuccessResponse(null, 'Exchange not found'), 404); } - logger.info('Exchange updated successfully', { + logger.info('Exchange updated successfully', { exchangeId, code: exchange.code, - updates: validatedUpdates + updates: validatedUpdates, }); - + // Log special actions if (validatedUpdates.visible === false) { - logger.warn('Exchange marked as hidden - provider mappings will be deleted', { + logger.warn('Exchange marked as hidden - provider mappings will be deleted', { exchangeId, - code: exchange.code + code: exchange.code, }); } @@ -124,7 +121,7 @@ exchangeRoutes.patch('/:id', async c => { // Get all provider mappings exchangeRoutes.get('/provider-mappings/all', async c => { logger.debug('Getting all provider mappings'); - + try { const mappings = await exchangeService.getAllProviderMappings(); logger.info('Successfully retrieved all provider mappings', { count: mappings.length }); @@ -139,18 +136,12 @@ exchangeRoutes.get('/provider-mappings/all', async c => { exchangeRoutes.get('/provider-mappings/:provider', async c => { const provider = c.req.param('provider'); logger.debug('Getting provider mappings by provider', { provider }); - + try { const mappings = await exchangeService.getProviderMappingsByProvider(provider); logger.info('Successfully retrieved provider mappings', { provider, count: mappings.length }); - - return c.json( - createSuccessResponse( - mappings, - undefined, - mappings.length - ) - ); + + return c.json(createSuccessResponse(mappings, undefined, mappings.length)); } catch (error) { logger.error('Failed to get provider mappings', { error, provider }); return handleError(c, error, 'to get provider mappings'); @@ -161,26 +152,26 @@ exchangeRoutes.get('/provider-mappings/:provider', async c => { exchangeRoutes.patch('/provider-mappings/:id', async c => { const mappingId = c.req.param('id'); logger.debug('Updating provider mapping', { mappingId }); - + try { const body = await c.req.json(); logger.debug('Received provider mapping update request', { mappingId, updates: body }); - + const validatedUpdates = validateUpdateProviderMapping(body); logger.debug('Provider mapping update data validated', { mappingId, validatedUpdates }); - + const mapping = await exchangeService.updateProviderMapping(mappingId, validatedUpdates); - + if (!mapping) { logger.warn('Provider mapping not found for update', { mappingId }); return c.json(createSuccessResponse(null, 'Provider mapping not found'), 404); } - logger.info('Provider mapping updated successfully', { + logger.info('Provider mapping updated successfully', { mappingId, provider: mapping.provider, providerExchangeCode: mapping.provider_exchange_code, - updates: validatedUpdates + updates: validatedUpdates, }); return c.json(createSuccessResponse(mapping, 'Provider mapping updated successfully')); @@ -193,26 +184,23 @@ exchangeRoutes.patch('/provider-mappings/:id', async c => { // Create new provider mapping exchangeRoutes.post('/provider-mappings', async c => { logger.debug('Creating new provider mapping'); - + try { const body = await c.req.json(); logger.debug('Received provider mapping creation request', { requestBody: body }); - + const validatedData = validateCreateProviderMapping(body); logger.debug('Provider mapping data validated successfully', { validatedData }); - + const mapping = await exchangeService.createProviderMapping(validatedData); - logger.info('Provider mapping created successfully', { + logger.info('Provider mapping created successfully', { mappingId: mapping.id, provider: mapping.provider, providerExchangeCode: mapping.provider_exchange_code, - masterExchangeId: mapping.master_exchange_id + masterExchangeId: mapping.master_exchange_id, }); - - return c.json( - createSuccessResponse(mapping, 'Provider mapping created successfully'), - 201 - ); + + return c.json(createSuccessResponse(mapping, 'Provider mapping created successfully'), 201); } catch (error) { logger.error('Failed to create provider mapping', { error }); return handleError(c, error, 'to create provider mapping'); @@ -222,7 +210,7 @@ exchangeRoutes.post('/provider-mappings', async c => { // Get all available providers exchangeRoutes.get('/providers/list', async c => { logger.debug('Getting providers list'); - + try { const providers = await exchangeService.getProviders(); logger.info('Successfully retrieved providers list', { count: providers.length, providers }); @@ -237,21 +225,15 @@ exchangeRoutes.get('/providers/list', async c => { exchangeRoutes.get('/provider-exchanges/unmapped/:provider', async c => { const provider = c.req.param('provider'); logger.debug('Getting unmapped provider exchanges', { provider }); - + try { const exchanges = await exchangeService.getUnmappedProviderExchanges(provider); - logger.info('Successfully retrieved unmapped provider exchanges', { - provider, - count: exchanges.length + logger.info('Successfully retrieved unmapped provider exchanges', { + provider, + count: exchanges.length, }); - - return c.json( - createSuccessResponse( - exchanges, - undefined, - exchanges.length - ) - ); + + return c.json(createSuccessResponse(exchanges, undefined, exchanges.length)); } catch (error) { logger.error('Failed to get unmapped provider exchanges', { error, provider }); return handleError(c, error, 'to get unmapped provider exchanges'); @@ -261,7 +243,7 @@ exchangeRoutes.get('/provider-exchanges/unmapped/:provider', async c => { // Get exchange statistics exchangeRoutes.get('/stats/summary', async c => { logger.debug('Getting exchange statistics'); - + try { const stats = await exchangeService.getExchangeStats(); logger.info('Successfully retrieved exchange statistics', { stats }); @@ -270,4 +252,4 @@ exchangeRoutes.get('/stats/summary', async c => { logger.error('Failed to get exchange statistics', { error }); return handleError(c, error, 'to get exchange statistics'); } -}); \ No newline at end of file +}); diff --git a/apps/web-api/src/routes/health.routes.ts b/apps/web-api/src/routes/health.routes.ts index 9ace743..2e36638 100644 --- a/apps/web-api/src/routes/health.routes.ts +++ b/apps/web-api/src/routes/health.routes.ts @@ -3,7 +3,7 @@ */ import { Hono } from 'hono'; import { getLogger } from '@stock-bot/logger'; -import { getPostgreSQLClient, getMongoDBClient } from '../clients'; +import { getMongoDBClient, getPostgreSQLClient } from '../clients'; const logger = getLogger('health-routes'); export const healthRoutes = new Hono(); @@ -11,13 +11,13 @@ export const healthRoutes = new Hono(); // Basic health check healthRoutes.get('/', c => { logger.debug('Basic health check requested'); - + const response = { status: 'healthy', service: 'web-api', timestamp: new Date().toISOString(), }; - + logger.info('Basic health check successful', { status: response.status }); return c.json(response); }); @@ -25,7 +25,7 @@ healthRoutes.get('/', c => { // Detailed health check with database connectivity healthRoutes.get('/detailed', async c => { logger.debug('Detailed health check requested'); - + const health = { status: 'healthy', service: 'web-api', @@ -80,19 +80,19 @@ healthRoutes.get('/detailed', async c => { health.status = allHealthy ? 'healthy' : 'unhealthy'; const statusCode = allHealthy ? 200 : 503; - + if (allHealthy) { logger.info('Detailed health check successful - all systems healthy', { mongodb: health.checks.mongodb.status, - postgresql: health.checks.postgresql.status + postgresql: health.checks.postgresql.status, }); } else { logger.warn('Detailed health check failed - some systems unhealthy', { mongodb: health.checks.mongodb.status, postgresql: health.checks.postgresql.status, - overallStatus: health.status + overallStatus: health.status, }); } - + return c.json(health, statusCode); }); diff --git a/apps/web-api/src/services/exchange.service.ts b/apps/web-api/src/services/exchange.service.ts index 8acbb0e..95eafad 100644 --- a/apps/web-api/src/services/exchange.service.ts +++ b/apps/web-api/src/services/exchange.service.ts @@ -1,15 +1,15 @@ import { getLogger } from '@stock-bot/logger'; -import { getPostgreSQLClient, getMongoDBClient } from '../clients'; +import { getMongoDBClient, getPostgreSQLClient } from '../clients'; import { - Exchange, - ExchangeWithMappings, - ProviderMapping, CreateExchangeRequest, - UpdateExchangeRequest, CreateProviderMappingRequest, - UpdateProviderMappingRequest, - ProviderExchange, + Exchange, ExchangeStats, + ExchangeWithMappings, + ProviderExchange, + ProviderMapping, + UpdateExchangeRequest, + UpdateProviderMappingRequest, } from '../types/exchange.types'; const logger = getLogger('exchange-service'); @@ -18,7 +18,7 @@ export class ExchangeService { private get postgresClient() { return getPostgreSQLClient(); } - + private get mongoClient() { return getMongoDBClient(); } @@ -63,14 +63,17 @@ export class ExchangeService { const mappingsResult = await this.postgresClient.query(mappingsQuery); // Group mappings by exchange ID - const mappingsByExchange = mappingsResult.rows.reduce((acc, mapping) => { - const exchangeId = mapping.master_exchange_id; - if (!acc[exchangeId]) { - acc[exchangeId] = []; - } - acc[exchangeId].push(mapping); - return acc; - }, {} as Record); + const mappingsByExchange = mappingsResult.rows.reduce( + (acc, mapping) => { + const exchangeId = mapping.master_exchange_id; + if (!acc[exchangeId]) { + acc[exchangeId] = []; + } + acc[exchangeId].push(mapping); + return acc; + }, + {} as Record + ); // Attach mappings to exchanges return exchangesResult.rows.map(exchange => ({ @@ -79,7 +82,9 @@ export class ExchangeService { })); } - async getExchangeById(id: string): Promise<{ exchange: Exchange; provider_mappings: ProviderMapping[] } | null> { + async getExchangeById( + id: string + ): Promise<{ exchange: Exchange; provider_mappings: ProviderMapping[] } | null> { const exchangeQuery = 'SELECT * FROM exchanges WHERE id = $1 AND visible = true'; const exchangeResult = await this.postgresClient.query(exchangeQuery, [id]); @@ -230,7 +235,10 @@ export class ExchangeService { return result.rows[0]; } - async updateProviderMapping(id: string, updates: UpdateProviderMappingRequest): Promise { + async updateProviderMapping( + id: string, + updates: UpdateProviderMappingRequest + ): Promise { const updateFields = []; const values = []; let paramIndex = 1; @@ -359,7 +367,6 @@ export class ExchangeService { break; } - default: throw new Error(`Unknown provider: ${provider}`); } @@ -369,4 +376,4 @@ export class ExchangeService { } // Export singleton instance -export const exchangeService = new ExchangeService(); \ No newline at end of file +export const exchangeService = new ExchangeService(); diff --git a/apps/web-api/src/types/exchange.types.ts b/apps/web-api/src/types/exchange.types.ts index d755efe..a367db7 100644 --- a/apps/web-api/src/types/exchange.types.ts +++ b/apps/web-api/src/types/exchange.types.ts @@ -100,4 +100,4 @@ export interface ApiResponse { error?: string; message?: string; total?: number; -} \ No newline at end of file +} diff --git a/apps/web-api/src/utils/error-handler.ts b/apps/web-api/src/utils/error-handler.ts index 77787e0..eba907d 100644 --- a/apps/web-api/src/utils/error-handler.ts +++ b/apps/web-api/src/utils/error-handler.ts @@ -1,7 +1,7 @@ import { Context } from 'hono'; import { getLogger } from '@stock-bot/logger'; -import { ValidationError } from './validation'; import { ApiResponse } from '../types/exchange.types'; +import { ValidationError } from './validation'; const logger = getLogger('error-handler'); @@ -61,4 +61,4 @@ export function createSuccessResponse( } return response; -} \ No newline at end of file +} diff --git a/apps/web-api/src/utils/validation.ts b/apps/web-api/src/utils/validation.ts index 47f4dfe..59441ec 100644 --- a/apps/web-api/src/utils/validation.ts +++ b/apps/web-api/src/utils/validation.ts @@ -1,7 +1,10 @@ import { CreateExchangeRequest, CreateProviderMappingRequest } from '../types/exchange.types'; export class ValidationError extends Error { - constructor(message: string, public field?: string) { + constructor( + message: string, + public field?: string + ) { super(message); this.name = 'ValidationError'; } @@ -38,7 +41,10 @@ export function validateCreateExchange(data: unknown): CreateExchangeRequest { } if (currency.length !== 3) { - throw new ValidationError('Currency must be exactly 3 characters (e.g., USD, EUR, CAD)', 'currency'); + throw new ValidationError( + 'Currency must be exactly 3 characters (e.g., USD, EUR, CAD)', + 'currency' + ); } return { @@ -172,4 +178,4 @@ export function validateUpdateProviderMapping(data: unknown): Record([]); @@ -62,18 +62,15 @@ export function useExchanges() { [fetchExchanges] ); - const fetchExchangeDetails = useCallback( - async (id: string): Promise => { - try { - return await exchangeApi.getExchangeById(id); - } catch (err) { - // Error fetching exchange details - error state will show in UI - setError(err instanceof Error ? err.message : 'Failed to fetch exchange details'); - return null; - } - }, - [] - ); + const fetchExchangeDetails = useCallback(async (id: string): Promise => { + try { + return await exchangeApi.getExchangeById(id); + } catch (err) { + // Error fetching exchange details - error state will show in UI + setError(err instanceof Error ? err.message : 'Failed to fetch exchange details'); + return null; + } + }, []); const fetchStats = useCallback(async (): Promise => { try { diff --git a/apps/web-app/src/features/exchanges/hooks/useFormValidation.ts b/apps/web-app/src/features/exchanges/hooks/useFormValidation.ts index 41f4bf1..5b2d2f3 100644 --- a/apps/web-app/src/features/exchanges/hooks/useFormValidation.ts +++ b/apps/web-app/src/features/exchanges/hooks/useFormValidation.ts @@ -1,22 +1,22 @@ -import { useState, useCallback } from 'react'; +import { useCallback, useState } from 'react'; import { FormErrors } from '../types'; -export function useFormValidation( - initialData: T, - validateFn: (data: T) => FormErrors -) { +export function useFormValidation(initialData: T, validateFn: (data: T) => FormErrors) { const [formData, setFormData] = useState(initialData); const [errors, setErrors] = useState({}); const [isSubmitting, setIsSubmitting] = useState(false); - const updateField = useCallback((field: keyof T, value: T[keyof T]) => { - setFormData(prev => ({ ...prev, [field]: value })); - - // Clear error when user starts typing - if (errors[field as string]) { - setErrors(prev => ({ ...prev, [field as string]: '' })); - } - }, [errors]); + const updateField = useCallback( + (field: keyof T, value: T[keyof T]) => { + setFormData(prev => ({ ...prev, [field]: value })); + + // Clear error when user starts typing + if (errors[field as string]) { + setErrors(prev => ({ ...prev, [field as string]: '' })); + } + }, + [errors] + ); const validate = useCallback((): boolean => { const newErrors = validateFn(formData); @@ -30,24 +30,29 @@ export function useFormValidation( setIsSubmitting(false); }, [initialData]); - const handleSubmit = useCallback(async ( - onSubmit: (data: T) => Promise, - onSuccess?: () => void, - onError?: (error: unknown) => void - ) => { - if (!validate()) {return;} + const handleSubmit = useCallback( + async ( + onSubmit: (data: T) => Promise, + onSuccess?: () => void, + onError?: (error: unknown) => void + ) => { + if (!validate()) { + return; + } - setIsSubmitting(true); - try { - await onSubmit(formData); - reset(); - onSuccess?.(); - } catch (error) { - onError?.(error); - } finally { - setIsSubmitting(false); - } - }, [formData, validate, reset]); + setIsSubmitting(true); + try { + await onSubmit(formData); + reset(); + onSuccess?.(); + } catch (error) { + onError?.(error); + } finally { + setIsSubmitting(false); + } + }, + [formData, validate, reset] + ); return { formData, @@ -59,4 +64,4 @@ export function useFormValidation( handleSubmit, setIsSubmitting, }; -} \ No newline at end of file +} diff --git a/apps/web-app/src/features/exchanges/services/exchangeApi.ts b/apps/web-app/src/features/exchanges/services/exchangeApi.ts index 3a416dc..fb3d7c9 100644 --- a/apps/web-app/src/features/exchanges/services/exchangeApi.ts +++ b/apps/web-app/src/features/exchanges/services/exchangeApi.ts @@ -1,25 +1,22 @@ import { ApiResponse, + CreateExchangeRequest, + CreateProviderMappingRequest, Exchange, ExchangeDetails, ExchangeStats, - ProviderMapping, ProviderExchange, - CreateExchangeRequest, + ProviderMapping, UpdateExchangeRequest, - CreateProviderMappingRequest, UpdateProviderMappingRequest, } from '../types'; const API_BASE_URL = import.meta.env.VITE_API_BASE_URL || 'http://localhost:4000/api'; class ExchangeApiService { - private async request( - endpoint: string, - options?: RequestInit - ): Promise> { + private async request(endpoint: string, options?: RequestInit): Promise> { const url = `${API_BASE_URL}${endpoint}`; - + const response = await fetch(url, { headers: { 'Content-Type': 'application/json', @@ -33,7 +30,7 @@ class ExchangeApiService { } const data = await response.json(); - + if (!data.success) { throw new Error(data.error || 'API request failed'); } @@ -76,10 +73,10 @@ class ExchangeApiService { // Provider Mappings async getProviderMappings(provider?: string): Promise { - const endpoint = provider + const endpoint = provider ? `/exchanges/provider-mappings/${provider}` : '/exchanges/provider-mappings/all'; - + const response = await this.request(endpoint); return response.data || []; } @@ -96,7 +93,7 @@ class ExchangeApiService { } async updateProviderMapping( - id: string, + id: string, data: UpdateProviderMappingRequest ): Promise { const response = await this.request(`/exchanges/provider-mappings/${id}`, { @@ -132,4 +129,4 @@ class ExchangeApiService { } // Export singleton instance -export const exchangeApi = new ExchangeApiService(); \ No newline at end of file +export const exchangeApi = new ExchangeApiService(); diff --git a/apps/web-app/src/features/exchanges/types/api.types.ts b/apps/web-app/src/features/exchanges/types/api.types.ts index e86503b..342536c 100644 --- a/apps/web-app/src/features/exchanges/types/api.types.ts +++ b/apps/web-app/src/features/exchanges/types/api.types.ts @@ -66,4 +66,4 @@ export interface ExchangeStats { active_provider_mappings: string; verified_provider_mappings: string; providers: string; -} \ No newline at end of file +} diff --git a/apps/web-app/src/features/exchanges/types/component.types.ts b/apps/web-app/src/features/exchanges/types/component.types.ts index ac22733..d6af0d6 100644 --- a/apps/web-app/src/features/exchanges/types/component.types.ts +++ b/apps/web-app/src/features/exchanges/types/component.types.ts @@ -32,7 +32,9 @@ export interface AddExchangeDialogProps extends BaseDialogProps { export interface AddProviderMappingDialogProps extends BaseDialogProps { exchangeId: string; exchangeName: string; - onCreateMapping: (request: import('./request.types').CreateProviderMappingRequest) => Promise; + onCreateMapping: ( + request: import('./request.types').CreateProviderMappingRequest + ) => Promise; } export interface DeleteExchangeDialogProps extends BaseDialogProps { @@ -40,4 +42,4 @@ export interface DeleteExchangeDialogProps extends BaseDialogProps { exchangeName: string; providerMappingCount: number; onConfirmDelete: (exchangeId: string) => Promise; -} \ No newline at end of file +} diff --git a/apps/web-app/src/features/exchanges/types/request.types.ts b/apps/web-app/src/features/exchanges/types/request.types.ts index 6624bb0..efd1553 100644 --- a/apps/web-app/src/features/exchanges/types/request.types.ts +++ b/apps/web-app/src/features/exchanges/types/request.types.ts @@ -32,4 +32,4 @@ export interface UpdateProviderMappingRequest { verified?: boolean; confidence?: number; master_exchange_id?: string; -} \ No newline at end of file +} diff --git a/apps/web-app/src/features/exchanges/utils/formatters.ts b/apps/web-app/src/features/exchanges/utils/formatters.ts index 7c7f36c..2572c33 100644 --- a/apps/web-app/src/features/exchanges/utils/formatters.ts +++ b/apps/web-app/src/features/exchanges/utils/formatters.ts @@ -21,7 +21,7 @@ export function sortProviderMappings(mappings: ProviderMapping[]): ProviderMappi if (!a.active && b.active) { return 1; } - + // Then by provider name return a.provider.localeCompare(b.provider); }); @@ -32,4 +32,4 @@ export function truncateText(text: string, maxLength: number): string { return text; } return text.substring(0, maxLength) + '...'; -} \ No newline at end of file +} diff --git a/apps/web-app/src/features/exchanges/utils/validation.ts b/apps/web-app/src/features/exchanges/utils/validation.ts index fd2dfdd..61e03ee 100644 --- a/apps/web-app/src/features/exchanges/utils/validation.ts +++ b/apps/web-app/src/features/exchanges/utils/validation.ts @@ -35,4 +35,4 @@ export function validateExchangeForm(data: { export function hasValidationErrors(errors: FormErrors): boolean { return Object.keys(errors).length > 0; -} \ No newline at end of file +} diff --git a/apps/web-app/src/lib/utils.ts b/apps/web-app/src/lib/utils.ts index be7fef6..db2551b 100644 --- a/apps/web-app/src/lib/utils.ts +++ b/apps/web-app/src/lib/utils.ts @@ -19,7 +19,11 @@ export function formatPercentage(value: number): string { } export function getValueColor(value: number): string { - if (value > 0) {return 'text-success';} - if (value < 0) {return 'text-danger';} + if (value > 0) { + return 'text-success'; + } + if (value < 0) { + return 'text-danger'; + } return 'text-text-secondary'; } diff --git a/apps/web-app/src/lib/utils/index.ts b/apps/web-app/src/lib/utils/index.ts index e365e72..483f89a 100644 --- a/apps/web-app/src/lib/utils/index.ts +++ b/apps/web-app/src/lib/utils/index.ts @@ -23,9 +23,15 @@ export function formatPercentage(value: number, decimals = 2): string { * Format large numbers with K, M, B suffixes */ export function formatNumber(num: number): string { - if (num >= 1e9) {return (num / 1e9).toFixed(1) + 'B';} - if (num >= 1e6) {return (num / 1e6).toFixed(1) + 'M';} - if (num >= 1e3) {return (num / 1e3).toFixed(1) + 'K';} + if (num >= 1e9) { + return (num / 1e9).toFixed(1) + 'B'; + } + if (num >= 1e6) { + return (num / 1e6).toFixed(1) + 'M'; + } + if (num >= 1e3) { + return (num / 1e3).toFixed(1) + 'K'; + } return num.toString(); } @@ -33,8 +39,12 @@ export function formatNumber(num: number): string { * Get color class based on numeric value (profit/loss) */ export function getValueColor(value: number): string { - if (value > 0) {return 'text-success';} - if (value < 0) {return 'text-danger';} + if (value > 0) { + return 'text-success'; + } + if (value < 0) { + return 'text-danger'; + } return 'text-text-secondary'; } @@ -42,6 +52,8 @@ export function getValueColor(value: number): string { * Truncate text to specified length */ export function truncateText(text: string, length: number): string { - if (text.length <= length) {return text;} + if (text.length <= length) { + return text; + } return text.slice(0, length) + '...'; } diff --git a/docs/enhanced-cache-usage.md b/docs/enhanced-cache-usage.md deleted file mode 100644 index c01a13f..0000000 --- a/docs/enhanced-cache-usage.md +++ /dev/null @@ -1,148 +0,0 @@ -# Enhanced Cache Provider Usage - -The Redis cache provider now supports advanced TTL handling and conditional operations. - -## Basic Usage (Backward Compatible) - -```typescript -import { RedisCache } from '@stock-bot/cache'; - -const cache = new RedisCache({ - keyPrefix: 'trading:', - defaultTTL: 3600 // 1 hour -}); - -// Simple set with TTL (old way - still works) -await cache.set('user:123', userData, 1800); // 30 minutes - -// Simple get -const user = await cache.get('user:123'); -``` - -## Enhanced Set Options - -```typescript -// Preserve existing TTL when updating -await cache.set('user:123', updatedUserData, { preserveTTL: true }); - -// Only set if key exists (update operation) -const oldValue = await cache.set('user:123', newData, { - onlyIfExists: true, - getOldValue: true -}); - -// Only set if key doesn't exist (create operation) -await cache.set('user:456', newUser, { - onlyIfNotExists: true, - ttl: 7200 // 2 hours -}); - -// Get old value when setting new one -const previousData = await cache.set('session:abc', sessionData, { - getOldValue: true, - ttl: 1800 -}); -``` - -## Convenience Methods - -```typescript -// Update value preserving TTL -await cache.update('user:123', updatedUserData); - -// Set only if exists -const updated = await cache.setIfExists('user:123', newData, 3600); - -// Set only if not exists (returns true if created) -const created = await cache.setIfNotExists('user:456', userData); - -// Replace existing key with new TTL -const oldData = await cache.replace('user:123', newData, 7200); - -// Atomic field updates -await cache.updateField('counter:views', (current) => (current || 0) + 1); - -await cache.updateField('user:123', (user) => ({ - ...user, - lastSeen: new Date().toISOString(), - loginCount: (user?.loginCount || 0) + 1 -})); -``` - -## Stock Bot Use Cases - -### 1. Rate Limiting -```typescript -// Only create rate limit if not exists -const rateLimited = await cache.setIfNotExists( - `ratelimit:${userId}:${endpoint}`, - { count: 1, resetTime: Date.now() + 60000 }, - 60 // 1 minute -); - -if (!rateLimited) { - // Increment existing counter - await cache.updateField(`ratelimit:${userId}:${endpoint}`, (data) => ({ - ...data, - count: data.count + 1 - })); -} -``` - -### 2. Session Management -```typescript -// Update session data without changing expiration -await cache.update(`session:${sessionId}`, { - ...sessionData, - lastActivity: Date.now() -}); -``` - -### 3. Cache Warming -```typescript -// Only update existing cached data, don't create new entries -const warmed = await cache.setIfExists(`stock:${symbol}:price`, latestPrice); -if (warmed) { - console.log(`Warmed cache for ${symbol}`); -} -``` - -### 4. Atomic Counters -```typescript -// Thread-safe counter increments -await cache.updateField('metrics:api:calls', (count) => (count || 0) + 1); -await cache.updateField('metrics:errors:500', (count) => (count || 0) + 1); -``` - -### 5. TTL Preservation for Frequently Updated Data -```typescript -// Keep original expiration when updating frequently changing data -await cache.set(`portfolio:${userId}:positions`, positions, { preserveTTL: true }); -``` - -## Error Handling - -The cache provider includes robust error handling: - -```typescript -try { - await cache.set('key', value); -} catch (error) { - // Errors are logged and fallback values returned - // The cache operations are non-blocking -} - -// Check cache health -const isHealthy = await cache.health(); - -// Wait for cache to be ready -await cache.waitForReady(10000); // 10 second timeout -``` - -## Performance Benefits - -1. **Atomic Operations**: `updateField` uses Lua scripts to prevent race conditions -2. **TTL Preservation**: Avoids unnecessary TTL resets on updates -3. **Conditional Operations**: Reduces network round trips -4. **Shared Connections**: Efficient connection pooling -5. **Error Recovery**: Graceful degradation when Redis is unavailable diff --git a/docs/loki-logging.md b/docs/loki-logging.md deleted file mode 100644 index 5ea7241..0000000 --- a/docs/loki-logging.md +++ /dev/null @@ -1,169 +0,0 @@ -# Loki Logging for Stock Bot - -This document outlines how to use the Loki logging system integrated with the Stock Bot platform (Updated June 2025). - -## Overview - -Loki provides centralized logging for all Stock Bot services with: - -1. **Centralized logging** for all microservices -2. **Log aggregation** and filtering by service, level, and custom labels -3. **Grafana integration** for visualization and dashboards -4. **Query capabilities** using LogQL for log analysis -5. **Alert capabilities** for critical issues - -## Getting Started - -### Starting the Logging Stack - -```cmd -# Start the monitoring stack (includes Loki and Grafana) -scripts\docker.ps1 monitoring -``` - -Or start services individually: - -```cmd -# Start Loki service only -docker-compose up -d loki - -# Start Loki and Grafana -docker-compose up -d loki grafana -``` - -### Viewing Logs - -Once started: - -1. Access Grafana at http://localhost:3000 (login with admin/admin) -2. Navigate to the "Stock Bot Logs" dashboard -3. View and query your logs - -## Using the Logger in Your Services - -The Stock Bot logger automatically sends logs to Loki using the updated pattern: - -```typescript -import { getLogger } from '@stock-bot/logger'; - -// Create a logger for your service -const logger = getLogger('your-service-name'); - -// Log at different levels -logger.debug('Detailed information for debugging'); -logger.info('General information about operations'); -logger.warn('Potential issues that don\'t affect operation'); -logger.error('Critical errors that require attention'); - -// Log with structured data (searchable in Loki) -logger.info('Processing trade', { - symbol: 'MSFT', - price: 410.75, - quantity: 50 -}); -``` - -## Configuration Options - -Logger configuration is managed through the `@stock-bot/config` package and can be set in your `.env` file: - -```bash -# Logging configuration -LOG_LEVEL=debug # debug, info, warn, error -LOG_CONSOLE=true # Log to console in addition to Loki -LOKI_HOST=localhost # Loki server hostname -LOKI_PORT=3100 # Loki server port -LOKI_RETENTION_DAYS=30 # Days to retain logs -LOKI_LABELS=environment=development,service=stock-bot # Default labels -LOKI_BATCH_SIZE=100 # Number of logs to batch before sending -LOKI_BATCH_WAIT=5 # Max time to wait before sending logs -``` - -## Useful Loki Queries - -Inside Grafana, you can use these LogQL queries to analyze your logs: - -1. **All logs from a specific service**: - ``` - {service="market-data-gateway"} - ``` - -2. **All error logs across all services**: - ``` - {level="error"} - ``` - -3. **Logs containing specific text**: - ``` - {service="market-data-gateway"} |= "trade" - ``` - -4. **Count of error logs by service over time**: - ``` - sum by(service) (count_over_time({level="error"}[5m])) - ``` - -## Testing the Logging Integration - -Test the logging integration using Bun: - -```cmd -# Run from project root using Bun (current runtime) -bun run tools/test-loki-logging.ts -``` - -## Architecture - -Our logging implementation follows this architecture: - -``` -┌─────────────────┐ ┌─────────────────┐ -│ Trading Services│────►│ @stock-bot/logger│ -└─────────────────┘ │ getLogger() │ - └────────┬────────┘ - │ - ▼ -┌────────────────────────────────────────┐ -│ Loki │ -└────────────────┬───────────────────────┘ - │ - ▼ -┌────────────────────────────────────────┐ -│ Grafana │ -└────────────────────────────────────────┘ -``` - -## Adding New Dashboards - -To create new Grafana dashboards for log visualization: - -1. Build your dashboard in the Grafana UI -2. Export it to JSON -3. Add it to `monitoring/grafana/provisioning/dashboards/json/` -4. Restart the monitoring stack - -## Troubleshooting - -If logs aren't appearing in Grafana: - -1. Run the status check script to verify Loki and Grafana are working: - ```cmd - tools\check-loki-status.bat - ``` - -2. Check that Loki and Grafana containers are running: - ```cmd - docker ps | findstr "loki grafana" - ``` - -3. Verify .env configuration for Loki host and port: - ```cmd - type .env | findstr "LOKI_" - ``` - -4. Ensure your service has the latest @stock-bot/logger package - -5. Check for errors in the Loki container logs: - ```cmd - docker logs stock-bot-loki - ``` diff --git a/docs/mongodb-multi-database-migration.md b/docs/mongodb-multi-database-migration.md deleted file mode 100644 index 30d6069..0000000 --- a/docs/mongodb-multi-database-migration.md +++ /dev/null @@ -1,212 +0,0 @@ -# MongoDB Client Multi-Database Migration Guide - -## Overview -Your MongoDB client has been enhanced to support multiple databases dynamically while maintaining full backward compatibility. - -## Key Features Added - -### 1. **Dynamic Database Switching** -```typescript -// Set default database (all operations will use this unless overridden) -client.setDefaultDatabase('analytics'); - -// Get current default database -const currentDb = client.getDefaultDatabase(); // Returns: 'analytics' -``` - -### 2. **Database Parameter in Methods** -All methods now accept an optional `database` parameter: - -```typescript -// Old way (still works - uses default database) -await client.batchUpsert('symbols', data, 'symbol'); - -// New way (specify database explicitly) -await client.batchUpsert('symbols', data, 'symbol', { database: 'stock' }); -``` - -### 3. **Convenience Methods** -Pre-configured methods for common databases: - -```typescript -// Stock database operations -await client.batchUpsertStock('symbols', data, 'symbol'); - -// Analytics database operations -await client.batchUpsertAnalytics('metrics', data, 'metric_name'); - -// Trading documents database operations -await client.batchUpsertTrading('orders', data, 'order_id'); -``` - -### 4. **Direct Database Access** -```typescript -// Get specific database instances -const stockDb = client.getDatabase('stock'); -const analyticsDb = client.getDatabase('analytics'); - -// Get collections with database override -const collection = client.getCollection('symbols', 'stock'); -``` - -## Migration Steps - -### Step 1: No Changes Required (Backward Compatible) -Your existing code continues to work unchanged: - -```typescript -// This still works exactly as before -const client = MongoDBClient.getInstance(); -await client.connect(); -await client.batchUpsert('exchanges', exchangeData, 'exchange_id'); -``` - -### Step 2: Organize Data by Database (Recommended) -Update your data service to use appropriate databases: - -```typescript -// In your data service initialization -export class DataService { - private mongoClient = MongoDBClient.getInstance(); - - async initialize() { - await this.mongoClient.connect(); - - // Set stock as default for most operations - this.mongoClient.setDefaultDatabase('stock'); - } - - async saveInteractiveBrokersData(exchanges: any[], symbols: any[]) { - // Stock market data goes to 'stock' database (default) - await this.mongoClient.batchUpsert('exchanges', exchanges, 'exchange_id'); - await this.mongoClient.batchUpsert('symbols', symbols, 'symbol'); - } - - async saveAnalyticsData(performance: any[]) { - // Analytics data goes to 'analytics' database - await this.mongoClient.batchUpsert( - 'performance', - performance, - 'date', - { database: 'analytics' } - ); - } -} -``` - -### Step 3: Use Convenience Methods (Optional) -Replace explicit database parameters with convenience methods: - -```typescript -// Instead of: -await client.batchUpsert('symbols', data, 'symbol', { database: 'stock' }); - -// Use: -await client.batchUpsertStock('symbols', data, 'symbol'); -``` - -## Factory Functions -New factory functions are available for easier database management: - -```typescript -import { - connectMongoDB, - setDefaultDatabase, - getCurrentDatabase, - getDatabase -} from '@stock-bot/mongodb-client'; - -// Set default database globally -setDefaultDatabase('analytics'); - -// Get current default -const current = getCurrentDatabase(); - -// Get specific database -const stockDb = getDatabase('stock'); -``` - -## Database Recommendations - -### Stock Database (`stock`) -- Market data (symbols, exchanges, prices) -- Financial instruments -- Market events -- Real-time data - -### Analytics Database (`analytics`) -- Performance metrics -- Calculated indicators -- Reports and dashboards -- Aggregated data - -### Trading Documents Database (`trading_documents`) -- Trade orders and executions -- User portfolios -- Transaction logs -- Audit trails - -## Example: Updating Your Data Service - -```typescript -// Before (still works) -export class DataService { - async saveExchanges(exchanges: any[]) { - const client = MongoDBClient.getInstance(); - await client.batchUpsert('exchanges', exchanges, 'exchange_id'); - } -} - -// After (recommended) -export class DataService { - private mongoClient = MongoDBClient.getInstance(); - - async initialize() { - await this.mongoClient.connect(); - this.mongoClient.setDefaultDatabase('stock'); // Set appropriate default - } - - async saveExchanges(exchanges: any[]) { - // Uses default 'stock' database - await this.mongoClient.batchUpsert('exchanges', exchanges, 'exchange_id'); - - // Or use convenience method - await this.mongoClient.batchUpsertStock('exchanges', exchanges, 'exchange_id'); - } - - async savePerformanceMetrics(metrics: any[]) { - // Save to analytics database - await this.mongoClient.batchUpsertAnalytics('metrics', metrics, 'metric_name'); - } -} -``` - -## Testing -Your existing tests continue to work. For new multi-database features: - -```typescript -import { MongoDBClient } from '@stock-bot/mongodb-client'; - -const client = MongoDBClient.getInstance(); -await client.connect(); - -// Test database switching -client.setDefaultDatabase('test_db'); -expect(client.getDefaultDatabase()).toBe('test_db'); - -// Test explicit database parameter -await client.batchUpsert('test_collection', data, 'id', { database: 'other_db' }); -``` - -## Benefits -1. **Organized Data**: Separate databases for different data types -2. **Better Performance**: Smaller, focused databases -3. **Easier Maintenance**: Clear data boundaries -4. **Scalability**: Can scale databases independently -5. **Backward Compatibility**: No breaking changes - -## Next Steps -1. Update your data service to use appropriate default database -2. Gradually migrate to using specific databases for different data types -3. Consider using convenience methods for cleaner code -4. Update tests to cover multi-database scenarios diff --git a/libs/core/config/config/default.json b/libs/core/config/config/default.json index 6314b96..10ce440 100644 --- a/libs/core/config/config/default.json +++ b/libs/core/config/config/default.json @@ -91,4 +91,4 @@ "apiKey": "", "apiUrl": "https://proxy.webshare.io/api/v2/" } -} \ No newline at end of file +} diff --git a/libs/core/config/config/development.json b/libs/core/config/config/development.json index 839c7e9..991c30c 100644 --- a/libs/core/config/config/development.json +++ b/libs/core/config/config/development.json @@ -45,4 +45,4 @@ "webmasterId": "" } } -} \ No newline at end of file +} diff --git a/libs/core/config/config/production.json b/libs/core/config/config/production.json index fe7a792..390d5ff 100644 --- a/libs/core/config/config/production.json +++ b/libs/core/config/config/production.json @@ -29,4 +29,4 @@ "retries": 5, "retryDelay": 2000 } -} \ No newline at end of file +} diff --git a/libs/core/config/config/test.json b/libs/core/config/config/test.json index 0cf6dcb..85f7ac4 100644 --- a/libs/core/config/config/test.json +++ b/libs/core/config/config/test.json @@ -39,4 +39,4 @@ "timeout": 5000, "retries": 1 } -} \ No newline at end of file +} diff --git a/libs/core/config/src/cli.ts b/libs/core/config/src/cli.ts index d3cfced..8abebef 100644 --- a/libs/core/config/src/cli.ts +++ b/libs/core/config/src/cli.ts @@ -1,196 +1,193 @@ -#!/usr/bin/env bun -/* eslint-disable no-console */ -import { parseArgs } from 'util'; -import { join } from 'path'; -import { ConfigManager } from './config-manager'; -import { appConfigSchema } from './schemas'; -import { - validateConfig, - formatValidationResult, - checkDeprecations, - checkRequiredEnvVars, - validateCompleteness -} from './utils/validation'; -import { redactSecrets } from './utils/secrets'; -import type { Environment } from './types'; - -interface CliOptions { - config?: string; - env?: string; - validate?: boolean; - show?: boolean; - check?: boolean; - json?: boolean; - help?: boolean; -} - -const DEPRECATIONS = { - 'service.legacyMode': 'Use service.mode instead', - 'database.redis': 'Use database.dragonfly instead', -}; - -const REQUIRED_PATHS = [ - 'service.name', - 'service.port', - 'database.postgres.host', - 'database.postgres.database', -]; - -const REQUIRED_ENV_VARS = [ - 'NODE_ENV', -]; - -const SECRET_PATHS = [ - 'database.postgres.password', - 'database.mongodb.uri', - 'providers.quoteMedia.apiKey', - 'providers.interactiveBrokers.clientId', -]; - -function printUsage() { - console.log(` -Stock Bot Configuration CLI - -Usage: bun run config-cli [options] - -Options: - --config Path to config directory (default: ./config) - --env Environment to use (development, test, production) - --validate Validate configuration against schema - --show Show current configuration (secrets redacted) - --check Run all configuration checks - --json Output in JSON format - --help Show this help message - -Examples: - # Validate configuration - bun run config-cli --validate - - # Show configuration for production - bun run config-cli --env production --show - - # Run all checks - bun run config-cli --check - - # Output configuration as JSON - bun run config-cli --show --json -`); -} - -async function main() { - const { values } = parseArgs({ - args: process.argv.slice(2), - options: { - config: { type: 'string' }, - env: { type: 'string' }, - validate: { type: 'boolean' }, - show: { type: 'boolean' }, - check: { type: 'boolean' }, - json: { type: 'boolean' }, - help: { type: 'boolean' }, - }, - }) as { values: CliOptions }; - - if (values.help) { - printUsage(); - process.exit(0); - } - - const configPath = values.config || join(process.cwd(), 'config'); - const environment = values.env as Environment; - - try { - const manager = new ConfigManager({ - configPath, - environment, - }); - - const config = await manager.initialize(appConfigSchema); - - if (values.validate) { - const result = validateConfig(config, appConfigSchema); - - if (values.json) { - console.log(JSON.stringify(result, null, 2)); - } else { - console.log(formatValidationResult(result)); - } - - process.exit(result.valid ? 0 : 1); - } - - if (values.show) { - const redacted = redactSecrets(config, SECRET_PATHS); - - if (values.json) { - console.log(JSON.stringify(redacted, null, 2)); - } else { - console.log('Current Configuration:'); - console.log(JSON.stringify(redacted, null, 2)); - } - } - - if (values.check) { - console.log('Running configuration checks...\n'); - - // Schema validation - console.log('1. Schema Validation:'); - const schemaResult = validateConfig(config, appConfigSchema); - console.log(formatValidationResult(schemaResult)); - console.log(); - - // Environment variables - console.log('2. Required Environment Variables:'); - const envResult = checkRequiredEnvVars(REQUIRED_ENV_VARS); - console.log(formatValidationResult(envResult)); - console.log(); - - // Required paths - console.log('3. Required Configuration Paths:'); - const pathResult = validateCompleteness(config, REQUIRED_PATHS); - console.log(formatValidationResult(pathResult)); - console.log(); - - // Deprecations - console.log('4. Deprecation Warnings:'); - const warnings = checkDeprecations(config, DEPRECATIONS); - if (warnings && warnings.length > 0) { - for (const warning of warnings) { - console.log(` ⚠️ ${warning.path}: ${warning.message}`); - } - } else { - console.log(' ✅ No deprecated options found'); - } - console.log(); - - // Overall result - const allValid = schemaResult.valid && envResult.valid && pathResult.valid; - - if (allValid) { - console.log('✅ All configuration checks passed!'); - process.exit(0); - } else { - console.log('❌ Some configuration checks failed'); - process.exit(1); - } - } - - if (!values.validate && !values.show && !values.check) { - console.log('No action specified. Use --help for usage information.'); - process.exit(1); - } - - } catch (error) { - if (values.json) { - console.error(JSON.stringify({ error: String(error) })); - } else { - console.error('Error:', error); - } - process.exit(1); - } -} - -// Run CLI -if (import.meta.main) { - main(); -} \ No newline at end of file +#!/usr/bin/env bun +/* eslint-disable no-console */ +import { join } from 'path'; +import { parseArgs } from 'util'; +import { redactSecrets } from './utils/secrets'; +import { + checkDeprecations, + checkRequiredEnvVars, + formatValidationResult, + validateCompleteness, + validateConfig, +} from './utils/validation'; +import { ConfigManager } from './config-manager'; +import { appConfigSchema } from './schemas'; +import type { Environment } from './types'; + +interface CliOptions { + config?: string; + env?: string; + validate?: boolean; + show?: boolean; + check?: boolean; + json?: boolean; + help?: boolean; +} + +const DEPRECATIONS = { + 'service.legacyMode': 'Use service.mode instead', + 'database.redis': 'Use database.dragonfly instead', +}; + +const REQUIRED_PATHS = [ + 'service.name', + 'service.port', + 'database.postgres.host', + 'database.postgres.database', +]; + +const REQUIRED_ENV_VARS = ['NODE_ENV']; + +const SECRET_PATHS = [ + 'database.postgres.password', + 'database.mongodb.uri', + 'providers.quoteMedia.apiKey', + 'providers.interactiveBrokers.clientId', +]; + +function printUsage() { + console.log(` +Stock Bot Configuration CLI + +Usage: bun run config-cli [options] + +Options: + --config Path to config directory (default: ./config) + --env Environment to use (development, test, production) + --validate Validate configuration against schema + --show Show current configuration (secrets redacted) + --check Run all configuration checks + --json Output in JSON format + --help Show this help message + +Examples: + # Validate configuration + bun run config-cli --validate + + # Show configuration for production + bun run config-cli --env production --show + + # Run all checks + bun run config-cli --check + + # Output configuration as JSON + bun run config-cli --show --json +`); +} + +async function main() { + const { values } = parseArgs({ + args: process.argv.slice(2), + options: { + config: { type: 'string' }, + env: { type: 'string' }, + validate: { type: 'boolean' }, + show: { type: 'boolean' }, + check: { type: 'boolean' }, + json: { type: 'boolean' }, + help: { type: 'boolean' }, + }, + }) as { values: CliOptions }; + + if (values.help) { + printUsage(); + process.exit(0); + } + + const configPath = values.config || join(process.cwd(), 'config'); + const environment = values.env as Environment; + + try { + const manager = new ConfigManager({ + configPath, + environment, + }); + + const config = await manager.initialize(appConfigSchema); + + if (values.validate) { + const result = validateConfig(config, appConfigSchema); + + if (values.json) { + console.log(JSON.stringify(result, null, 2)); + } else { + console.log(formatValidationResult(result)); + } + + process.exit(result.valid ? 0 : 1); + } + + if (values.show) { + const redacted = redactSecrets(config, SECRET_PATHS); + + if (values.json) { + console.log(JSON.stringify(redacted, null, 2)); + } else { + console.log('Current Configuration:'); + console.log(JSON.stringify(redacted, null, 2)); + } + } + + if (values.check) { + console.log('Running configuration checks...\n'); + + // Schema validation + console.log('1. Schema Validation:'); + const schemaResult = validateConfig(config, appConfigSchema); + console.log(formatValidationResult(schemaResult)); + console.log(); + + // Environment variables + console.log('2. Required Environment Variables:'); + const envResult = checkRequiredEnvVars(REQUIRED_ENV_VARS); + console.log(formatValidationResult(envResult)); + console.log(); + + // Required paths + console.log('3. Required Configuration Paths:'); + const pathResult = validateCompleteness(config, REQUIRED_PATHS); + console.log(formatValidationResult(pathResult)); + console.log(); + + // Deprecations + console.log('4. Deprecation Warnings:'); + const warnings = checkDeprecations(config, DEPRECATIONS); + if (warnings && warnings.length > 0) { + for (const warning of warnings) { + console.log(` ⚠️ ${warning.path}: ${warning.message}`); + } + } else { + console.log(' ✅ No deprecated options found'); + } + console.log(); + + // Overall result + const allValid = schemaResult.valid && envResult.valid && pathResult.valid; + + if (allValid) { + console.log('✅ All configuration checks passed!'); + process.exit(0); + } else { + console.log('❌ Some configuration checks failed'); + process.exit(1); + } + } + + if (!values.validate && !values.show && !values.check) { + console.log('No action specified. Use --help for usage information.'); + process.exit(1); + } + } catch (error) { + if (values.json) { + console.error(JSON.stringify({ error: String(error) })); + } else { + console.error('Error:', error); + } + process.exit(1); + } +} + +// Run CLI +if (import.meta.main) { + main(); +} diff --git a/libs/core/config/src/errors.ts b/libs/core/config/src/errors.ts index a0d4bee..dd5fa31 100644 --- a/libs/core/config/src/errors.ts +++ b/libs/core/config/src/errors.ts @@ -6,15 +6,21 @@ export class ConfigError extends Error { } export class ConfigValidationError extends ConfigError { - constructor(message: string, public errors: unknown) { + constructor( + message: string, + public errors: unknown + ) { super(message); this.name = 'ConfigValidationError'; } } export class ConfigLoaderError extends ConfigError { - constructor(message: string, public loader: string) { + constructor( + message: string, + public loader: string + ) { super(`${loader}: ${message}`); this.name = 'ConfigLoaderError'; } -} \ No newline at end of file +} diff --git a/libs/core/config/src/schemas/base.schema.ts b/libs/core/config/src/schemas/base.schema.ts index 2adb6bc..1695553 100644 --- a/libs/core/config/src/schemas/base.schema.ts +++ b/libs/core/config/src/schemas/base.schema.ts @@ -7,4 +7,4 @@ export const baseConfigSchema = z.object({ name: z.string().optional(), version: z.string().optional(), debug: z.boolean().default(false), -}); \ No newline at end of file +}); diff --git a/libs/core/config/src/schemas/database.schema.ts b/libs/core/config/src/schemas/database.schema.ts index f4d1aff..c2c0c78 100644 --- a/libs/core/config/src/schemas/database.schema.ts +++ b/libs/core/config/src/schemas/database.schema.ts @@ -61,4 +61,4 @@ export const databaseConfigSchema = z.object({ questdb: questdbConfigSchema, mongodb: mongodbConfigSchema, dragonfly: dragonflyConfigSchema, -}); \ No newline at end of file +}); diff --git a/libs/core/config/src/schemas/index.ts b/libs/core/config/src/schemas/index.ts index eed8827..0ea6635 100644 --- a/libs/core/config/src/schemas/index.ts +++ b/libs/core/config/src/schemas/index.ts @@ -1,87 +1,105 @@ -export * from './base.schema'; -export * from './database.schema'; -export * from './provider.schema'; -export * from './service.schema'; - import { z } from 'zod'; import { baseConfigSchema, environmentSchema } from './base.schema'; import { providerConfigSchema, webshareProviderConfigSchema } from './provider.schema'; import { httpConfigSchema, queueConfigSchema } from './service.schema'; +export * from './base.schema'; +export * from './database.schema'; +export * from './provider.schema'; +export * from './service.schema'; + // Flexible service schema with defaults -const flexibleServiceConfigSchema = z.object({ - name: z.string().default('default-service'), - port: z.number().min(1).max(65535).default(3000), - host: z.string().default('0.0.0.0'), - healthCheckPath: z.string().default('/health'), - metricsPath: z.string().default('/metrics'), - shutdownTimeout: z.number().default(30000), - cors: z.object({ - enabled: z.boolean().default(true), - origin: z.union([z.string(), z.array(z.string())]).default('*'), - credentials: z.boolean().default(true), - }).default({}), -}).default({}); +const flexibleServiceConfigSchema = z + .object({ + name: z.string().default('default-service'), + port: z.number().min(1).max(65535).default(3000), + host: z.string().default('0.0.0.0'), + healthCheckPath: z.string().default('/health'), + metricsPath: z.string().default('/metrics'), + shutdownTimeout: z.number().default(30000), + cors: z + .object({ + enabled: z.boolean().default(true), + origin: z.union([z.string(), z.array(z.string())]).default('*'), + credentials: z.boolean().default(true), + }) + .default({}), + }) + .default({}); // Flexible database schema with defaults -const flexibleDatabaseConfigSchema = z.object({ - postgres: z.object({ - host: z.string().default('localhost'), - port: z.number().default(5432), - database: z.string().default('test_db'), - user: z.string().default('test_user'), - password: z.string().default('test_pass'), - ssl: z.boolean().default(false), - poolSize: z.number().min(1).max(100).default(10), - connectionTimeout: z.number().default(30000), - idleTimeout: z.number().default(10000), - }).default({}), - questdb: z.object({ - host: z.string().default('localhost'), - ilpPort: z.number().default(9009), - httpPort: z.number().default(9000), - pgPort: z.number().default(8812), - database: z.string().default('questdb'), - user: z.string().default('admin'), - password: z.string().default('quest'), - bufferSize: z.number().default(65536), - flushInterval: z.number().default(1000), - }).default({}), - mongodb: z.object({ - uri: z.string().url().optional(), - host: z.string().default('localhost'), - port: z.number().default(27017), - database: z.string().default('test_mongo'), - user: z.string().optional(), - password: z.string().optional(), - authSource: z.string().default('admin'), - replicaSet: z.string().optional(), - poolSize: z.number().min(1).max(100).default(10), - }).default({}), - dragonfly: z.object({ - host: z.string().default('localhost'), - port: z.number().default(6379), - password: z.string().optional(), - db: z.number().min(0).max(15).default(0), - keyPrefix: z.string().optional(), - ttl: z.number().optional(), - maxRetries: z.number().default(3), - retryDelay: z.number().default(100), - }).default({}), -}).default({}); +const flexibleDatabaseConfigSchema = z + .object({ + postgres: z + .object({ + host: z.string().default('localhost'), + port: z.number().default(5432), + database: z.string().default('test_db'), + user: z.string().default('test_user'), + password: z.string().default('test_pass'), + ssl: z.boolean().default(false), + poolSize: z.number().min(1).max(100).default(10), + connectionTimeout: z.number().default(30000), + idleTimeout: z.number().default(10000), + }) + .default({}), + questdb: z + .object({ + host: z.string().default('localhost'), + ilpPort: z.number().default(9009), + httpPort: z.number().default(9000), + pgPort: z.number().default(8812), + database: z.string().default('questdb'), + user: z.string().default('admin'), + password: z.string().default('quest'), + bufferSize: z.number().default(65536), + flushInterval: z.number().default(1000), + }) + .default({}), + mongodb: z + .object({ + uri: z.string().url().optional(), + host: z.string().default('localhost'), + port: z.number().default(27017), + database: z.string().default('test_mongo'), + user: z.string().optional(), + password: z.string().optional(), + authSource: z.string().default('admin'), + replicaSet: z.string().optional(), + poolSize: z.number().min(1).max(100).default(10), + }) + .default({}), + dragonfly: z + .object({ + host: z.string().default('localhost'), + port: z.number().default(6379), + password: z.string().optional(), + db: z.number().min(0).max(15).default(0), + keyPrefix: z.string().optional(), + ttl: z.number().optional(), + maxRetries: z.number().default(3), + retryDelay: z.number().default(100), + }) + .default({}), + }) + .default({}); // Flexible log schema with defaults (renamed from logging) -const flexibleLogConfigSchema = z.object({ - level: z.enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal']).default('info'), - format: z.enum(['json', 'pretty']).default('json'), - hideObject: z.boolean().default(false), - loki: z.object({ - enabled: z.boolean().default(false), - host: z.string().default('localhost'), - port: z.number().default(3100), - labels: z.record(z.string()).default({}), - }).optional(), -}).default({}); +const flexibleLogConfigSchema = z + .object({ + level: z.enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal']).default('info'), + format: z.enum(['json', 'pretty']).default('json'), + hideObject: z.boolean().default(false), + loki: z + .object({ + enabled: z.boolean().default(false), + host: z.string().default('localhost'), + port: z.number().default(3100), + labels: z.record(z.string()).default({}), + }) + .optional(), + }) + .default({}); // Complete application configuration schema export const appConfigSchema = baseConfigSchema.extend({ @@ -95,4 +113,4 @@ export const appConfigSchema = baseConfigSchema.extend({ webshare: webshareProviderConfigSchema.optional(), }); -export type AppConfig = z.infer; \ No newline at end of file +export type AppConfig = z.infer; diff --git a/libs/core/config/src/schemas/provider.schema.ts b/libs/core/config/src/schemas/provider.schema.ts index b4bdbce..62ccf72 100644 --- a/libs/core/config/src/schemas/provider.schema.ts +++ b/libs/core/config/src/schemas/provider.schema.ts @@ -5,10 +5,12 @@ export const baseProviderConfigSchema = z.object({ name: z.string(), enabled: z.boolean().default(true), priority: z.number().default(0), - rateLimit: z.object({ - maxRequests: z.number().default(100), - windowMs: z.number().default(60000), - }).optional(), + rateLimit: z + .object({ + maxRequests: z.number().default(100), + windowMs: z.number().default(60000), + }) + .optional(), timeout: z.number().default(30000), retries: z.number().default(3), }); @@ -71,4 +73,4 @@ export const providerSchemas = { qm: qmProviderConfigSchema, yahoo: yahooProviderConfigSchema, webshare: webshareProviderConfigSchema, -} as const; \ No newline at end of file +} as const; diff --git a/libs/core/config/src/schemas/service.schema.ts b/libs/core/config/src/schemas/service.schema.ts index 5268c85..d0c0c89 100644 --- a/libs/core/config/src/schemas/service.schema.ts +++ b/libs/core/config/src/schemas/service.schema.ts @@ -8,23 +8,27 @@ export const serviceConfigSchema = z.object({ healthCheckPath: z.string().default('/health'), metricsPath: z.string().default('/metrics'), shutdownTimeout: z.number().default(30000), - cors: z.object({ - enabled: z.boolean().default(true), - origin: z.union([z.string(), z.array(z.string())]).default('*'), - credentials: z.boolean().default(true), - }).default({}), + cors: z + .object({ + enabled: z.boolean().default(true), + origin: z.union([z.string(), z.array(z.string())]).default('*'), + credentials: z.boolean().default(true), + }) + .default({}), }); // Logging configuration export const loggingConfigSchema = z.object({ level: z.enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal']).default('info'), format: z.enum(['json', 'pretty']).default('json'), - loki: z.object({ - enabled: z.boolean().default(false), - host: z.string().default('localhost'), - port: z.number().default(3100), - labels: z.record(z.string()).default({}), - }).optional(), + loki: z + .object({ + enabled: z.boolean().default(false), + host: z.string().default('localhost'), + port: z.number().default(3100), + labels: z.record(z.string()).default({}), + }) + .optional(), }); // Queue configuration @@ -35,15 +39,19 @@ export const queueConfigSchema = z.object({ password: z.string().optional(), db: z.number().default(1), }), - defaultJobOptions: z.object({ - attempts: z.number().default(3), - backoff: z.object({ - type: z.enum(['exponential', 'fixed']).default('exponential'), - delay: z.number().default(1000), - }).default({}), - removeOnComplete: z.number().default(10), - removeOnFail: z.number().default(5), - }).default({}), + defaultJobOptions: z + .object({ + attempts: z.number().default(3), + backoff: z + .object({ + type: z.enum(['exponential', 'fixed']).default('exponential'), + delay: z.number().default(1000), + }) + .default({}), + removeOnComplete: z.number().default(10), + removeOnFail: z.number().default(5), + }) + .default({}), }); // HTTP client configuration @@ -52,12 +60,16 @@ export const httpConfigSchema = z.object({ retries: z.number().default(3), retryDelay: z.number().default(1000), userAgent: z.string().optional(), - proxy: z.object({ - enabled: z.boolean().default(false), - url: z.string().url().optional(), - auth: z.object({ - username: z.string(), - password: z.string(), - }).optional(), - }).optional(), -}); \ No newline at end of file + proxy: z + .object({ + enabled: z.boolean().default(false), + url: z.string().url().optional(), + auth: z + .object({ + username: z.string(), + password: z.string(), + }) + .optional(), + }) + .optional(), +}); diff --git a/libs/core/config/src/utils/secrets.ts b/libs/core/config/src/utils/secrets.ts index b4c57a2..d12fc86 100644 --- a/libs/core/config/src/utils/secrets.ts +++ b/libs/core/config/src/utils/secrets.ts @@ -1,183 +1,178 @@ -import { z } from 'zod'; - -/** - * Secret value wrapper to prevent accidental logging - */ -export class SecretValue { - private readonly value: T; - private readonly masked: string; - - constructor(value: T, mask: string = '***') { - this.value = value; - this.masked = mask; - } - - /** - * Get the actual secret value - * @param reason - Required reason for accessing the secret - */ - reveal(reason: string): T { - if (!reason) { - throw new Error('Reason required for revealing secret value'); - } - return this.value; - } - - /** - * Get masked representation - */ - toString(): string { - return this.masked; - } - - /** - * Prevent JSON serialization of actual value - */ - toJSON(): string { - return this.masked; - } - - /** - * Check if value matches without revealing it - */ - equals(other: T): boolean { - return this.value === other; - } - - /** - * Transform the secret value - */ - map(fn: (value: T) => R, reason: string): SecretValue { - return new SecretValue(fn(this.reveal(reason))); - } -} - -/** - * Zod schema for secret values - */ -export const secretSchema = (_schema: T) => { - return z.custom>>( - (val) => val instanceof SecretValue, - { - message: 'Expected SecretValue instance', - } - ); -}; - -/** - * Transform string to SecretValue in Zod schema - */ -export const secretStringSchema = z - .string() - .transform((val) => new SecretValue(val)); - -/** - * Create a secret value - */ -export function secret(value: T, mask?: string): SecretValue { - return new SecretValue(value, mask); -} - -/** - * Check if a value is a secret - */ -export function isSecret(value: unknown): value is SecretValue { - return value instanceof SecretValue; -} - -/** - * Redact secrets from an object - */ -export function redactSecrets>( - obj: T, - secretPaths: string[] = [] -): T { - const result = { ...obj }; - - // Redact known secret paths - for (const path of secretPaths) { - const keys = path.split('.'); - let current: any = result; - - for (let i = 0; i < keys.length - 1; i++) { - const key = keys[i]; - if (key && current[key] && typeof current[key] === 'object') { - current = current[key]; - } else { - break; - } - } - - const lastKey = keys[keys.length - 1]; - if (current && lastKey && lastKey in current) { - current[lastKey] = '***REDACTED***'; - } - } - - // Recursively redact SecretValue instances - function redactSecretValues(obj: any): any { - if (obj === null || obj === undefined) { - return obj; - } - - if (isSecret(obj)) { - return obj.toString(); - } - - if (Array.isArray(obj)) { - return obj.map(redactSecretValues); - } - - if (typeof obj === 'object') { - const result: any = {}; - for (const [key, value] of Object.entries(obj)) { - result[key] = redactSecretValues(value); - } - return result; - } - - return obj; - } - - return redactSecretValues(result); -} - -/** - * Environment variable names that should be treated as secrets - */ -export const COMMON_SECRET_PATTERNS = [ - /password/i, - /secret/i, - /key/i, - /token/i, - /credential/i, - /private/i, - /auth/i, - /api[-_]?key/i, -]; - -/** - * Check if an environment variable name indicates a secret - */ -export function isSecretEnvVar(name: string): boolean { - return COMMON_SECRET_PATTERNS.some(pattern => pattern.test(name)); -} - -/** - * Wrap environment variables that look like secrets - */ -export function wrapSecretEnvVars( - env: Record -): Record { - const result: Record = {}; - - for (const [key, value] of Object.entries(env)) { - if (value !== undefined && isSecretEnvVar(key)) { - result[key] = new SecretValue(value, `***${key}***`); - } else { - result[key] = value; - } - } - - return result; -} \ No newline at end of file +import { z } from 'zod'; + +/** + * Secret value wrapper to prevent accidental logging + */ +export class SecretValue { + private readonly value: T; + private readonly masked: string; + + constructor(value: T, mask: string = '***') { + this.value = value; + this.masked = mask; + } + + /** + * Get the actual secret value + * @param reason - Required reason for accessing the secret + */ + reveal(reason: string): T { + if (!reason) { + throw new Error('Reason required for revealing secret value'); + } + return this.value; + } + + /** + * Get masked representation + */ + toString(): string { + return this.masked; + } + + /** + * Prevent JSON serialization of actual value + */ + toJSON(): string { + return this.masked; + } + + /** + * Check if value matches without revealing it + */ + equals(other: T): boolean { + return this.value === other; + } + + /** + * Transform the secret value + */ + map(fn: (value: T) => R, reason: string): SecretValue { + return new SecretValue(fn(this.reveal(reason))); + } +} + +/** + * Zod schema for secret values + */ +export const secretSchema = (_schema: T) => { + return z.custom>>(val => val instanceof SecretValue, { + message: 'Expected SecretValue instance', + }); +}; + +/** + * Transform string to SecretValue in Zod schema + */ +export const secretStringSchema = z.string().transform(val => new SecretValue(val)); + +/** + * Create a secret value + */ +export function secret(value: T, mask?: string): SecretValue { + return new SecretValue(value, mask); +} + +/** + * Check if a value is a secret + */ +export function isSecret(value: unknown): value is SecretValue { + return value instanceof SecretValue; +} + +/** + * Redact secrets from an object + */ +export function redactSecrets>( + obj: T, + secretPaths: string[] = [] +): T { + const result = { ...obj }; + + // Redact known secret paths + for (const path of secretPaths) { + const keys = path.split('.'); + let current: any = result; + + for (let i = 0; i < keys.length - 1; i++) { + const key = keys[i]; + if (key && current[key] && typeof current[key] === 'object') { + current = current[key]; + } else { + break; + } + } + + const lastKey = keys[keys.length - 1]; + if (current && lastKey && lastKey in current) { + current[lastKey] = '***REDACTED***'; + } + } + + // Recursively redact SecretValue instances + function redactSecretValues(obj: any): any { + if (obj === null || obj === undefined) { + return obj; + } + + if (isSecret(obj)) { + return obj.toString(); + } + + if (Array.isArray(obj)) { + return obj.map(redactSecretValues); + } + + if (typeof obj === 'object') { + const result: any = {}; + for (const [key, value] of Object.entries(obj)) { + result[key] = redactSecretValues(value); + } + return result; + } + + return obj; + } + + return redactSecretValues(result); +} + +/** + * Environment variable names that should be treated as secrets + */ +export const COMMON_SECRET_PATTERNS = [ + /password/i, + /secret/i, + /key/i, + /token/i, + /credential/i, + /private/i, + /auth/i, + /api[-_]?key/i, +]; + +/** + * Check if an environment variable name indicates a secret + */ +export function isSecretEnvVar(name: string): boolean { + return COMMON_SECRET_PATTERNS.some(pattern => pattern.test(name)); +} + +/** + * Wrap environment variables that look like secrets + */ +export function wrapSecretEnvVars( + env: Record +): Record { + const result: Record = {}; + + for (const [key, value] of Object.entries(env)) { + if (value !== undefined && isSecretEnvVar(key)) { + result[key] = new SecretValue(value, `***${key}***`); + } else { + result[key] = value; + } + } + + return result; +} diff --git a/libs/core/config/src/utils/validation.ts b/libs/core/config/src/utils/validation.ts index 5b61b82..ece59c7 100644 --- a/libs/core/config/src/utils/validation.ts +++ b/libs/core/config/src/utils/validation.ts @@ -1,195 +1,188 @@ -import { z } from 'zod'; - -export interface ValidationResult { - valid: boolean; - errors?: Array<{ - path: string; - message: string; - expected?: string; - received?: string; - }>; - warnings?: Array<{ - path: string; - message: string; - }>; -} - -/** - * Validate configuration against a schema - */ -export function validateConfig( - config: unknown, - schema: z.ZodSchema -): ValidationResult { - try { - schema.parse(config); - return { valid: true }; - } catch (error) { - if (error instanceof z.ZodError) { - const errors = error.errors.map(err => ({ - path: err.path.join('.'), - message: err.message, - expected: 'expected' in err ? String(err.expected) : undefined, - received: 'received' in err ? String(err.received) : undefined, - })); - - return { valid: false, errors }; - } - - throw error; - } -} - -/** - * Check for deprecated configuration options - */ -export function checkDeprecations( - config: Record, - deprecations: Record -): ValidationResult['warnings'] { - const warnings: ValidationResult['warnings'] = []; - - function checkObject(obj: Record, path: string[] = []): void { - for (const [key, value] of Object.entries(obj)) { - const currentPath = [...path, key]; - const pathStr = currentPath.join('.'); - - if (pathStr in deprecations) { - const deprecationMessage = deprecations[pathStr]; - if (deprecationMessage) { - warnings?.push({ - path: pathStr, - message: deprecationMessage, - }); - } - } - - if (value && typeof value === 'object' && !Array.isArray(value)) { - checkObject(value as Record, currentPath); - } - } - } - - checkObject(config); - return warnings; -} - -/** - * Check for required environment variables - */ -export function checkRequiredEnvVars( - required: string[] -): ValidationResult { - const errors: ValidationResult['errors'] = []; - - for (const envVar of required) { - if (!process.env[envVar]) { - errors.push({ - path: `env.${envVar}`, - message: `Required environment variable ${envVar} is not set`, - }); - } - } - - return { - valid: errors.length === 0, - errors: errors.length > 0 ? errors : undefined, - }; -} - -/** - * Validate configuration completeness - */ -export function validateCompleteness( - config: Record, - required: string[] -): ValidationResult { - const errors: ValidationResult['errors'] = []; - - for (const path of required) { - const keys = path.split('.'); - let current: any = config; - let found = true; - - for (const key of keys) { - if (current && typeof current === 'object' && key in current) { - current = current[key]; - } else { - found = false; - break; - } - } - - if (!found || current === undefined || current === null) { - errors.push({ - path, - message: `Required configuration value is missing`, - }); - } - } - - return { - valid: errors.length === 0, - errors: errors.length > 0 ? errors : undefined, - }; -} - -/** - * Format validation result for display - */ -export function formatValidationResult(result: ValidationResult): string { - const lines: string[] = []; - - if (result.valid) { - lines.push('✅ Configuration is valid'); - } else { - lines.push('❌ Configuration validation failed'); - } - - if (result.errors && result.errors.length > 0) { - lines.push('\nErrors:'); - for (const error of result.errors) { - lines.push(` - ${error.path}: ${error.message}`); - if (error.expected && error.received) { - lines.push(` Expected: ${error.expected}, Received: ${error.received}`); - } - } - } - - if (result.warnings && result.warnings.length > 0) { - lines.push('\nWarnings:'); - for (const warning of result.warnings) { - lines.push(` - ${warning.path}: ${warning.message}`); - } - } - - return lines.join('\n'); -} - -/** - * Create a strict schema that doesn't allow extra properties - */ -export function createStrictSchema( - shape: T -): z.ZodObject { - return z.object(shape).strict(); -} - -/** - * Merge multiple schemas - */ -export function mergeSchemas( - ...schemas: T -): z.ZodIntersection { - if (schemas.length < 2) { - throw new Error('At least two schemas required for merge'); - } - - let result = schemas[0]!.and(schemas[1]!); - - for (let i = 2; i < schemas.length; i++) { - result = result.and(schemas[i]!) as any; - } - - return result as any; -} \ No newline at end of file +import { z } from 'zod'; + +export interface ValidationResult { + valid: boolean; + errors?: Array<{ + path: string; + message: string; + expected?: string; + received?: string; + }>; + warnings?: Array<{ + path: string; + message: string; + }>; +} + +/** + * Validate configuration against a schema + */ +export function validateConfig(config: unknown, schema: z.ZodSchema): ValidationResult { + try { + schema.parse(config); + return { valid: true }; + } catch (error) { + if (error instanceof z.ZodError) { + const errors = error.errors.map(err => ({ + path: err.path.join('.'), + message: err.message, + expected: 'expected' in err ? String(err.expected) : undefined, + received: 'received' in err ? String(err.received) : undefined, + })); + + return { valid: false, errors }; + } + + throw error; + } +} + +/** + * Check for deprecated configuration options + */ +export function checkDeprecations( + config: Record, + deprecations: Record +): ValidationResult['warnings'] { + const warnings: ValidationResult['warnings'] = []; + + function checkObject(obj: Record, path: string[] = []): void { + for (const [key, value] of Object.entries(obj)) { + const currentPath = [...path, key]; + const pathStr = currentPath.join('.'); + + if (pathStr in deprecations) { + const deprecationMessage = deprecations[pathStr]; + if (deprecationMessage) { + warnings?.push({ + path: pathStr, + message: deprecationMessage, + }); + } + } + + if (value && typeof value === 'object' && !Array.isArray(value)) { + checkObject(value as Record, currentPath); + } + } + } + + checkObject(config); + return warnings; +} + +/** + * Check for required environment variables + */ +export function checkRequiredEnvVars(required: string[]): ValidationResult { + const errors: ValidationResult['errors'] = []; + + for (const envVar of required) { + if (!process.env[envVar]) { + errors.push({ + path: `env.${envVar}`, + message: `Required environment variable ${envVar} is not set`, + }); + } + } + + return { + valid: errors.length === 0, + errors: errors.length > 0 ? errors : undefined, + }; +} + +/** + * Validate configuration completeness + */ +export function validateCompleteness( + config: Record, + required: string[] +): ValidationResult { + const errors: ValidationResult['errors'] = []; + + for (const path of required) { + const keys = path.split('.'); + let current: any = config; + let found = true; + + for (const key of keys) { + if (current && typeof current === 'object' && key in current) { + current = current[key]; + } else { + found = false; + break; + } + } + + if (!found || current === undefined || current === null) { + errors.push({ + path, + message: `Required configuration value is missing`, + }); + } + } + + return { + valid: errors.length === 0, + errors: errors.length > 0 ? errors : undefined, + }; +} + +/** + * Format validation result for display + */ +export function formatValidationResult(result: ValidationResult): string { + const lines: string[] = []; + + if (result.valid) { + lines.push('✅ Configuration is valid'); + } else { + lines.push('❌ Configuration validation failed'); + } + + if (result.errors && result.errors.length > 0) { + lines.push('\nErrors:'); + for (const error of result.errors) { + lines.push(` - ${error.path}: ${error.message}`); + if (error.expected && error.received) { + lines.push(` Expected: ${error.expected}, Received: ${error.received}`); + } + } + } + + if (result.warnings && result.warnings.length > 0) { + lines.push('\nWarnings:'); + for (const warning of result.warnings) { + lines.push(` - ${warning.path}: ${warning.message}`); + } + } + + return lines.join('\n'); +} + +/** + * Create a strict schema that doesn't allow extra properties + */ +export function createStrictSchema(shape: T): z.ZodObject { + return z.object(shape).strict(); +} + +/** + * Merge multiple schemas + */ +export function mergeSchemas( + ...schemas: T +): z.ZodIntersection { + if (schemas.length < 2) { + throw new Error('At least two schemas required for merge'); + } + + let result = schemas[0]!.and(schemas[1]!); + + for (let i = 2; i < schemas.length; i++) { + result = result.and(schemas[i]!) as any; + } + + return result as any; +} diff --git a/libs/core/config/test/config-manager.test.ts b/libs/core/config/test/config-manager.test.ts index bce9edb..62f04e7 100644 --- a/libs/core/config/test/config-manager.test.ts +++ b/libs/core/config/test/config-manager.test.ts @@ -1,215 +1,221 @@ -import { describe, test, expect, beforeEach } from 'bun:test'; -import { z } from 'zod'; -import { ConfigManager } from '../src/config-manager'; -import { ConfigLoader } from '../src/types'; -import { ConfigValidationError } from '../src/errors'; - -// Mock loader for testing -class MockLoader implements ConfigLoader { - priority = 0; - - constructor( - private data: Record, - public override priority: number = 0 - ) {} - - async load(): Promise> { - return this.data; - } -} - -// Test schema -const testSchema = z.object({ - app: z.object({ - name: z.string(), - version: z.string(), - port: z.number().positive(), - }), - database: z.object({ - host: z.string(), - port: z.number(), - }), - environment: z.enum(['development', 'test', 'production']), -}); - -type TestConfig = z.infer; - -describe('ConfigManager', () => { - let manager: ConfigManager; - - beforeEach(() => { - manager = new ConfigManager({ - loaders: [ - new MockLoader({ - app: { - name: 'test-app', - version: '1.0.0', - port: 3000, - }, - database: { - host: 'localhost', - port: 5432, - }, - }), - ], - environment: 'test', - }); - }); - - test('should initialize configuration', async () => { - const config = await manager.initialize(testSchema); - - expect(config.app.name).toBe('test-app'); - expect(config.app.version).toBe('1.0.0'); - expect(config.environment).toBe('test'); - }); - - test('should merge multiple loaders by priority', async () => { - manager = new ConfigManager({ - loaders: [ - new MockLoader({ app: { name: 'base', port: 3000 } }, 0), - new MockLoader({ app: { name: 'override', version: '2.0.0' } }, 10), - new MockLoader({ database: { host: 'prod-db' } }, 5), - ], - environment: 'test', - }); - - const config = await manager.initialize(); - - expect(config.app.name).toBe('override'); - expect(config.app.version).toBe('2.0.0'); - expect(config.app.port).toBe(3000); - expect(config.database.host).toBe('prod-db'); - }); - - test('should validate configuration with schema', async () => { - manager = new ConfigManager({ - loaders: [ - new MockLoader({ - app: { - name: 'test-app', - version: '1.0.0', - port: 'invalid', // Should be number - }, - }), - ], - }); - - await expect(manager.initialize(testSchema)).rejects.toThrow(ConfigValidationError); - }); - - test('should get configuration value by path', async () => { - await manager.initialize(testSchema); - - expect(manager.getValue('app.name')).toBe('test-app'); - expect(manager.getValue('database.port')).toBe(5432); - }); - - test('should check if configuration path exists', async () => { - await manager.initialize(testSchema); - - expect(manager.has('app.name')).toBe(true); - expect(manager.has('app.nonexistent')).toBe(false); - }); - - test('should update configuration at runtime', async () => { - await manager.initialize(testSchema); - - manager.set({ - app: { - name: 'updated-app', - }, - }); - - const config = manager.get(); - expect(config.app.name).toBe('updated-app'); - expect(config.app.version).toBe('1.0.0'); // Should preserve other values - }); - - test('should validate updates against schema', async () => { - await manager.initialize(testSchema); - - expect(() => { - manager.set({ - app: { - port: 'invalid' as any, - }, - }); - }).toThrow(ConfigValidationError); - }); - - test('should reset configuration', async () => { - await manager.initialize(testSchema); - manager.reset(); - - expect(() => manager.get()).toThrow('Configuration not initialized'); - }); - - test('should create typed getter', async () => { - await manager.initialize(testSchema); - - const appSchema = z.object({ - app: z.object({ - name: z.string(), - version: z.string(), - }), - }); - - const getAppConfig = manager.createTypedGetter(appSchema); - const appConfig = getAppConfig(); - - expect(appConfig.app.name).toBe('test-app'); - }); - - test('should detect environment correctly', () => { - const originalEnv = process.env.NODE_ENV; - - process.env.NODE_ENV = 'production'; - const prodManager = new ConfigManager({ loaders: [] }); - expect(prodManager.getEnvironment()).toBe('production'); - - process.env.NODE_ENV = 'test'; - const testManager = new ConfigManager({ loaders: [] }); - expect(testManager.getEnvironment()).toBe('test'); - - process.env.NODE_ENV = originalEnv; - }); - - test('should handle deep merge correctly', async () => { - manager = new ConfigManager({ - loaders: [ - new MockLoader({ - app: { - settings: { - feature1: true, - feature2: false, - nested: { - value: 'base', - }, - }, - }, - }, 0), - new MockLoader({ - app: { - settings: { - feature2: true, - feature3: true, - nested: { - value: 'override', - extra: 'new', - }, - }, - }, - }, 10), - ], - }); - - const config = await manager.initialize(); - - expect(config.app.settings.feature1).toBe(true); - expect(config.app.settings.feature2).toBe(true); - expect(config.app.settings.feature3).toBe(true); - expect(config.app.settings.nested.value).toBe('override'); - expect(config.app.settings.nested.extra).toBe('new'); - }); -}); \ No newline at end of file +import { beforeEach, describe, expect, test } from 'bun:test'; +import { z } from 'zod'; +import { ConfigManager } from '../src/config-manager'; +import { ConfigValidationError } from '../src/errors'; +import { ConfigLoader } from '../src/types'; + +// Mock loader for testing +class MockLoader implements ConfigLoader { + priority = 0; + + constructor( + private data: Record, + public override priority: number = 0 + ) {} + + async load(): Promise> { + return this.data; + } +} + +// Test schema +const testSchema = z.object({ + app: z.object({ + name: z.string(), + version: z.string(), + port: z.number().positive(), + }), + database: z.object({ + host: z.string(), + port: z.number(), + }), + environment: z.enum(['development', 'test', 'production']), +}); + +type TestConfig = z.infer; + +describe('ConfigManager', () => { + let manager: ConfigManager; + + beforeEach(() => { + manager = new ConfigManager({ + loaders: [ + new MockLoader({ + app: { + name: 'test-app', + version: '1.0.0', + port: 3000, + }, + database: { + host: 'localhost', + port: 5432, + }, + }), + ], + environment: 'test', + }); + }); + + test('should initialize configuration', async () => { + const config = await manager.initialize(testSchema); + + expect(config.app.name).toBe('test-app'); + expect(config.app.version).toBe('1.0.0'); + expect(config.environment).toBe('test'); + }); + + test('should merge multiple loaders by priority', async () => { + manager = new ConfigManager({ + loaders: [ + new MockLoader({ app: { name: 'base', port: 3000 } }, 0), + new MockLoader({ app: { name: 'override', version: '2.0.0' } }, 10), + new MockLoader({ database: { host: 'prod-db' } }, 5), + ], + environment: 'test', + }); + + const config = await manager.initialize(); + + expect(config.app.name).toBe('override'); + expect(config.app.version).toBe('2.0.0'); + expect(config.app.port).toBe(3000); + expect(config.database.host).toBe('prod-db'); + }); + + test('should validate configuration with schema', async () => { + manager = new ConfigManager({ + loaders: [ + new MockLoader({ + app: { + name: 'test-app', + version: '1.0.0', + port: 'invalid', // Should be number + }, + }), + ], + }); + + await expect(manager.initialize(testSchema)).rejects.toThrow(ConfigValidationError); + }); + + test('should get configuration value by path', async () => { + await manager.initialize(testSchema); + + expect(manager.getValue('app.name')).toBe('test-app'); + expect(manager.getValue('database.port')).toBe(5432); + }); + + test('should check if configuration path exists', async () => { + await manager.initialize(testSchema); + + expect(manager.has('app.name')).toBe(true); + expect(manager.has('app.nonexistent')).toBe(false); + }); + + test('should update configuration at runtime', async () => { + await manager.initialize(testSchema); + + manager.set({ + app: { + name: 'updated-app', + }, + }); + + const config = manager.get(); + expect(config.app.name).toBe('updated-app'); + expect(config.app.version).toBe('1.0.0'); // Should preserve other values + }); + + test('should validate updates against schema', async () => { + await manager.initialize(testSchema); + + expect(() => { + manager.set({ + app: { + port: 'invalid' as any, + }, + }); + }).toThrow(ConfigValidationError); + }); + + test('should reset configuration', async () => { + await manager.initialize(testSchema); + manager.reset(); + + expect(() => manager.get()).toThrow('Configuration not initialized'); + }); + + test('should create typed getter', async () => { + await manager.initialize(testSchema); + + const appSchema = z.object({ + app: z.object({ + name: z.string(), + version: z.string(), + }), + }); + + const getAppConfig = manager.createTypedGetter(appSchema); + const appConfig = getAppConfig(); + + expect(appConfig.app.name).toBe('test-app'); + }); + + test('should detect environment correctly', () => { + const originalEnv = process.env.NODE_ENV; + + process.env.NODE_ENV = 'production'; + const prodManager = new ConfigManager({ loaders: [] }); + expect(prodManager.getEnvironment()).toBe('production'); + + process.env.NODE_ENV = 'test'; + const testManager = new ConfigManager({ loaders: [] }); + expect(testManager.getEnvironment()).toBe('test'); + + process.env.NODE_ENV = originalEnv; + }); + + test('should handle deep merge correctly', async () => { + manager = new ConfigManager({ + loaders: [ + new MockLoader( + { + app: { + settings: { + feature1: true, + feature2: false, + nested: { + value: 'base', + }, + }, + }, + }, + 0 + ), + new MockLoader( + { + app: { + settings: { + feature2: true, + feature3: true, + nested: { + value: 'override', + extra: 'new', + }, + }, + }, + }, + 10 + ), + ], + }); + + const config = await manager.initialize(); + + expect(config.app.settings.feature1).toBe(true); + expect(config.app.settings.feature2).toBe(true); + expect(config.app.settings.feature3).toBe(true); + expect(config.app.settings.nested.value).toBe('override'); + expect(config.app.settings.nested.extra).toBe('new'); + }); +}); diff --git a/libs/core/config/test/dynamic-location.test.ts b/libs/core/config/test/dynamic-location.test.ts index b632d3d..c938599 100644 --- a/libs/core/config/test/dynamic-location.test.ts +++ b/libs/core/config/test/dynamic-location.test.ts @@ -1,10 +1,10 @@ -import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import { existsSync, mkdirSync, rmSync, writeFileSync } from 'fs'; import { join } from 'path'; -import { mkdirSync, writeFileSync, rmSync, existsSync } from 'fs'; +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import { ConfigManager } from '../src/config-manager'; -import { FileLoader } from '../src/loaders/file.loader'; -import { EnvLoader } from '../src/loaders/env.loader'; import { initializeConfig, initializeServiceConfig, resetConfig } from '../src/index'; +import { EnvLoader } from '../src/loaders/env.loader'; +import { FileLoader } from '../src/loaders/file.loader'; import { appConfigSchema } from '../src/schemas'; // Test directories setup @@ -23,33 +23,33 @@ describe('Dynamic Location Config Loading', () => { if (existsSync(TEST_ROOT)) { rmSync(TEST_ROOT, { recursive: true, force: true }); } - + // Reset config singleton resetConfig(); - + // Create test directory structure setupTestScenarios(); }); - + afterEach(() => { // Clean up test directories if (existsSync(TEST_ROOT)) { rmSync(TEST_ROOT, { recursive: true, force: true }); } - + // Reset config singleton resetConfig(); }); test('should load config from monorepo root', async () => { const originalCwd = process.cwd(); - + try { // Change to monorepo root process.chdir(SCENARIOS.monorepoRoot); - + const config = await initializeConfig(); - + expect(config.name).toBe('monorepo-root'); expect(config.version).toBe('1.0.0'); expect(config.database.postgres.host).toBe('localhost'); @@ -60,13 +60,13 @@ describe('Dynamic Location Config Loading', () => { test('should load config from app service directory', async () => { const originalCwd = process.cwd(); - + try { // Change to app service directory process.chdir(SCENARIOS.appService); - + const config = await initializeServiceConfig(); - + // Should inherit from root + override with service config expect(config.name).toBe('test-service'); // Overridden by service expect(config.version).toBe('1.0.0'); // From root @@ -79,13 +79,13 @@ describe('Dynamic Location Config Loading', () => { test('should load config from lib directory', async () => { const originalCwd = process.cwd(); - + try { // Change to lib directory process.chdir(SCENARIOS.libService); - + const config = await initializeServiceConfig(); - + // Should inherit from root + override with lib config expect(config.name).toBe('test-lib'); // Overridden by lib expect(config.version).toBe('2.0.0'); // Overridden by lib @@ -98,13 +98,13 @@ describe('Dynamic Location Config Loading', () => { test('should load config from deeply nested service', async () => { const originalCwd = process.cwd(); - + try { // Change to nested service directory process.chdir(SCENARIOS.nestedService); - + const config = await initializeServiceConfig(); - + // Should inherit from root + override with nested service config expect(config.name).toBe('deep-service'); // Overridden by nested service // NOTE: Version inheritance doesn't work for deeply nested services (3+ levels) @@ -119,13 +119,13 @@ describe('Dynamic Location Config Loading', () => { test('should load config from standalone project', async () => { const originalCwd = process.cwd(); - + try { // Change to standalone directory process.chdir(SCENARIOS.standalone); - + const config = await initializeConfig(); - + expect(config.name).toBe('standalone-app'); expect(config.version).toBe('0.1.0'); expect(config.database.postgres.host).toBe('standalone-db'); @@ -136,16 +136,16 @@ describe('Dynamic Location Config Loading', () => { test('should handle missing config files gracefully', async () => { const originalCwd = process.cwd(); - + try { // Change to directory with no config files const emptyDir = join(TEST_ROOT, 'empty'); mkdirSync(emptyDir, { recursive: true }); process.chdir(emptyDir); - + // Should not throw but use defaults and env vars const config = await initializeConfig(); - + // Should have default values from schema expect(config.environment).toBe('test'); // Tests run with NODE_ENV=test expect(typeof config.service).toBe('object'); @@ -157,18 +157,18 @@ describe('Dynamic Location Config Loading', () => { test('should prioritize environment variables over file configs', async () => { const originalCwd = process.cwd(); const originalEnv = { ...process.env }; - + try { // Set environment variables process.env.NAME = 'env-override'; process.env.VERSION = '3.0.0'; process.env.DATABASE_POSTGRES_HOST = 'env-db'; - + process.chdir(SCENARIOS.appService); - + resetConfig(); // Reset to test env override const config = await initializeServiceConfig(); - + // Environment variables should override file configs expect(config.name).toBe('env-override'); expect(config.version).toBe('3.0.0'); @@ -181,18 +181,18 @@ describe('Dynamic Location Config Loading', () => { test('should work with custom config paths', async () => { const originalCwd = process.cwd(); - + try { process.chdir(SCENARIOS.monorepoRoot); - + // Initialize with custom config path resetConfig(); const manager = new ConfigManager({ - configPath: join(SCENARIOS.appService, 'config') + configPath: join(SCENARIOS.appService, 'config'), }); - + const config = await manager.initialize(appConfigSchema); - + // Should load from the custom path expect(config.name).toBe('test-service'); expect(config.service.port).toBe(4000); @@ -217,7 +217,7 @@ function setupTestScenarios() { version: '1.0.0', service: { name: 'monorepo-root', - port: 3000 + port: 3000, }, database: { postgres: { @@ -225,32 +225,32 @@ function setupTestScenarios() { port: 5432, database: 'test_db', user: 'test_user', - password: 'test_pass' + password: 'test_pass', }, questdb: { host: 'localhost', - ilpPort: 9009 + ilpPort: 9009, }, mongodb: { host: 'localhost', port: 27017, - database: 'test_mongo' + database: 'test_mongo', }, dragonfly: { host: 'localhost', - port: 6379 - } + port: 6379, + }, }, logging: { - level: 'info' - } + level: 'info', + }, }; - + writeFileSync( join(SCENARIOS.monorepoRoot, 'config', 'development.json'), JSON.stringify(rootConfig, null, 2) ); - + writeFileSync( join(SCENARIOS.monorepoRoot, 'config', 'test.json'), JSON.stringify(rootConfig, null, 2) @@ -261,20 +261,20 @@ function setupTestScenarios() { name: 'test-service', database: { postgres: { - host: 'service-db' - } + host: 'service-db', + }, }, service: { name: 'test-service', - port: 4000 - } + port: 4000, + }, }; - + writeFileSync( join(SCENARIOS.appService, 'config', 'development.json'), JSON.stringify(appServiceConfig, null, 2) ); - + writeFileSync( join(SCENARIOS.appService, 'config', 'test.json'), JSON.stringify(appServiceConfig, null, 2) @@ -286,15 +286,15 @@ function setupTestScenarios() { version: '2.0.0', service: { name: 'test-lib', - port: 5000 - } + port: 5000, + }, }; - + writeFileSync( join(SCENARIOS.libService, 'config', 'development.json'), JSON.stringify(libServiceConfig, null, 2) ); - + writeFileSync( join(SCENARIOS.libService, 'config', 'test.json'), JSON.stringify(libServiceConfig, null, 2) @@ -305,20 +305,20 @@ function setupTestScenarios() { name: 'deep-service', database: { postgres: { - host: 'deep-db' - } + host: 'deep-db', + }, }, service: { name: 'deep-service', - port: 6000 - } + port: 6000, + }, }; - + writeFileSync( join(SCENARIOS.nestedService, 'config', 'development.json'), JSON.stringify(nestedServiceConfig, null, 2) ); - + writeFileSync( join(SCENARIOS.nestedService, 'config', 'test.json'), JSON.stringify(nestedServiceConfig, null, 2) @@ -330,7 +330,7 @@ function setupTestScenarios() { version: '0.1.0', service: { name: 'standalone-app', - port: 7000 + port: 7000, }, database: { postgres: { @@ -338,32 +338,32 @@ function setupTestScenarios() { port: 5432, database: 'standalone_db', user: 'standalone_user', - password: 'standalone_pass' + password: 'standalone_pass', }, questdb: { host: 'localhost', - ilpPort: 9009 + ilpPort: 9009, }, mongodb: { host: 'localhost', port: 27017, - database: 'standalone_mongo' + database: 'standalone_mongo', }, dragonfly: { host: 'localhost', - port: 6379 - } + port: 6379, + }, }, logging: { - level: 'debug' - } + level: 'debug', + }, }; - + writeFileSync( join(SCENARIOS.standalone, 'config', 'development.json'), JSON.stringify(standaloneConfig, null, 2) ); - + writeFileSync( join(SCENARIOS.standalone, 'config', 'test.json'), JSON.stringify(standaloneConfig, null, 2) @@ -383,4 +383,4 @@ DEBUG=true APP_EXTRA_FEATURE=enabled ` ); -} \ No newline at end of file +} diff --git a/libs/core/config/test/edge-cases.test.ts b/libs/core/config/test/edge-cases.test.ts index bae771a..12f6f7d 100644 --- a/libs/core/config/test/edge-cases.test.ts +++ b/libs/core/config/test/edge-cases.test.ts @@ -1,12 +1,12 @@ -import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import { chmodSync, existsSync, mkdirSync, rmSync, writeFileSync } from 'fs'; import { join } from 'path'; -import { mkdirSync, writeFileSync, rmSync, existsSync, chmodSync } from 'fs'; +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import { ConfigManager } from '../src/config-manager'; -import { FileLoader } from '../src/loaders/file.loader'; -import { EnvLoader } from '../src/loaders/env.loader'; -import { initializeConfig, initializeServiceConfig, resetConfig } from '../src/index'; -import { appConfigSchema } from '../src/schemas'; import { ConfigError, ConfigValidationError } from '../src/errors'; +import { initializeConfig, initializeServiceConfig, resetConfig } from '../src/index'; +import { EnvLoader } from '../src/loaders/env.loader'; +import { FileLoader } from '../src/loaders/file.loader'; +import { appConfigSchema } from '../src/schemas'; const TEST_DIR = join(__dirname, 'edge-case-tests'); @@ -17,9 +17,9 @@ describe('Edge Cases and Error Handling', () => { beforeEach(() => { originalEnv = { ...process.env }; originalCwd = process.cwd(); - + resetConfig(); - + if (existsSync(TEST_DIR)) { rmSync(TEST_DIR, { recursive: true, force: true }); } @@ -30,7 +30,7 @@ describe('Edge Cases and Error Handling', () => { process.env = originalEnv; process.chdir(originalCwd); resetConfig(); - + if (existsSync(TEST_DIR)) { rmSync(TEST_DIR, { recursive: true, force: true }); } @@ -39,7 +39,7 @@ describe('Edge Cases and Error Handling', () => { test('should handle missing .env files gracefully', async () => { // No .env file exists const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); // Should not throw even without .env file @@ -50,15 +50,12 @@ describe('Edge Cases and Error Handling', () => { test('should handle corrupted JSON config files', async () => { const configDir = join(TEST_DIR, 'config'); mkdirSync(configDir, { recursive: true }); - + // Create corrupted JSON file - writeFileSync( - join(configDir, 'development.json'), - '{ "app": { "name": "test", invalid json }' - ); + writeFileSync(join(configDir, 'development.json'), '{ "app": { "name": "test", invalid json }'); const manager = new ConfigManager({ - loaders: [new FileLoader(configDir, 'development')] + loaders: [new FileLoader(configDir, 'development')], }); // Should throw error for invalid JSON @@ -67,9 +64,9 @@ describe('Edge Cases and Error Handling', () => { test('should handle missing config directories', async () => { const nonExistentDir = join(TEST_DIR, 'nonexistent'); - + const manager = new ConfigManager({ - loaders: [new FileLoader(nonExistentDir, 'development')] + loaders: [new FileLoader(nonExistentDir, 'development')], }); // Should not throw, should return empty config @@ -80,16 +77,16 @@ describe('Edge Cases and Error Handling', () => { test('should handle permission denied on config files', async () => { const configDir = join(TEST_DIR, 'config'); mkdirSync(configDir, { recursive: true }); - + const configFile = join(configDir, 'development.json'); writeFileSync(configFile, JSON.stringify({ app: { name: 'test' } })); - + // Make file unreadable (this might not work on all systems) try { chmodSync(configFile, 0o000); - + const manager = new ConfigManager({ - loaders: [new FileLoader(configDir, 'development')] + loaders: [new FileLoader(configDir, 'development')], }); // Should handle permission error gracefully @@ -109,26 +106,23 @@ describe('Edge Cases and Error Handling', () => { // This tests deep merge with potential circular references const configDir = join(TEST_DIR, 'config'); mkdirSync(configDir, { recursive: true }); - + writeFileSync( join(configDir, 'development.json'), JSON.stringify({ app: { name: 'test', settings: { - ref: 'settings' - } - } + ref: 'settings', + }, + }, }) ); process.env.APP_SETTINGS_NESTED_VALUE = 'deep-value'; const manager = new ConfigManager({ - loaders: [ - new FileLoader(configDir, 'development'), - new EnvLoader('') - ] + loaders: [new FileLoader(configDir, 'development'), new EnvLoader('')], }); const config = await manager.initialize(appConfigSchema); @@ -138,13 +132,13 @@ describe('Edge Cases and Error Handling', () => { test('should handle extremely deep nesting in environment variables', async () => { // Test very deep nesting process.env.LEVEL1_LEVEL2_LEVEL3_LEVEL4_LEVEL5_VALUE = 'deep-value'; - + const manager = new ConfigManager({ - loaders: [new EnvLoader('', { nestedDelimiter: '_' })] + loaders: [new EnvLoader('', { nestedDelimiter: '_' })], }); const config = await manager.initialize(); - + // Should create nested structure expect((config as any).level1?.level2?.level3?.level4?.level5?.value).toBe('deep-value'); }); @@ -152,15 +146,15 @@ describe('Edge Cases and Error Handling', () => { test('should handle conflicting data types in config merging', async () => { const configDir = join(TEST_DIR, 'config'); mkdirSync(configDir, { recursive: true }); - + // File config has object writeFileSync( join(configDir, 'development.json'), JSON.stringify({ database: { host: 'localhost', - port: 5432 - } + port: 5432, + }, }) ); @@ -168,14 +162,11 @@ describe('Edge Cases and Error Handling', () => { process.env.DATABASE = 'simple-string'; const manager = new ConfigManager({ - loaders: [ - new FileLoader(configDir, 'development'), - new EnvLoader('') - ] + loaders: [new FileLoader(configDir, 'development'), new EnvLoader('')], }); const config = await manager.initialize(appConfigSchema); - + // Environment variable should win expect(config.database).toBe('simple-string'); }); @@ -184,15 +175,15 @@ describe('Edge Cases and Error Handling', () => { // Create multiple config setups in different directories const dir1 = join(TEST_DIR, 'dir1'); const dir2 = join(TEST_DIR, 'dir2'); - + mkdirSync(join(dir1, 'config'), { recursive: true }); mkdirSync(join(dir2, 'config'), { recursive: true }); - + writeFileSync( join(dir1, 'config', 'development.json'), JSON.stringify({ app: { name: 'dir1-app' } }) ); - + writeFileSync( join(dir2, 'config', 'development.json'), JSON.stringify({ app: { name: 'dir2-app' } }) @@ -229,13 +220,13 @@ JSON_VALUE={"key": "value", "nested": {"array": [1, 2, 3]}} ); process.chdir(TEST_DIR); - + const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); const config = await manager.initialize(); - + // Should handle valid entries expect(process.env.VALID_KEY).toBe('valid_value'); expect(process.env.KEY_WITH_QUOTES).toBe('quoted value'); @@ -245,12 +236,12 @@ JSON_VALUE={"key": "value", "nested": {"array": [1, 2, 3]}} test('should handle empty config files', async () => { const configDir = join(TEST_DIR, 'config'); mkdirSync(configDir, { recursive: true }); - + // Create empty JSON file writeFileSync(join(configDir, 'development.json'), '{}'); - + const manager = new ConfigManager({ - loaders: [new FileLoader(configDir, 'development')] + loaders: [new FileLoader(configDir, 'development')], }); const config = await manager.initialize(appConfigSchema); @@ -260,7 +251,7 @@ JSON_VALUE={"key": "value", "nested": {"array": [1, 2, 3]}} test('should handle config initialization without schema', async () => { const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); // Initialize without schema @@ -271,7 +262,7 @@ JSON_VALUE={"key": "value", "nested": {"array": [1, 2, 3]}} test('should handle accessing config before initialization', () => { const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); // Should throw error when accessing uninitialized config @@ -282,15 +273,15 @@ JSON_VALUE={"key": "value", "nested": {"array": [1, 2, 3]}} test('should handle invalid config paths in getValue', async () => { const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); const config = await manager.initialize(appConfigSchema); - + // Should throw for invalid paths expect(() => manager.getValue('nonexistent.path')).toThrow('Configuration key not found'); expect(() => manager.getValue('app.nonexistent')).toThrow('Configuration key not found'); - + // Should work for valid paths expect(() => manager.getValue('environment')).not.toThrow(); }); @@ -301,11 +292,11 @@ JSON_VALUE={"key": "value", "nested": {"array": [1, 2, 3]}} process.env.EMPTY_VALUE = ''; const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); const config = await manager.initialize(); - + expect((config as any).null_value).toBe(null); expect((config as any).undefined_value).toBe(undefined); expect((config as any).empty_value).toBe(''); @@ -318,7 +309,7 @@ JSON_VALUE={"key": "value", "nested": {"array": [1, 2, 3]}} process.env.SERVICE_PORT = 'not-a-number'; // This should cause validation to fail const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); await expect(manager.initialize(appConfigSchema)).rejects.toThrow(ConfigValidationError); @@ -326,7 +317,7 @@ JSON_VALUE={"key": "value", "nested": {"array": [1, 2, 3]}} test('should handle config updates with invalid schema', async () => { const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); await manager.initialize(appConfigSchema); @@ -335,8 +326,8 @@ JSON_VALUE={"key": "value", "nested": {"array": [1, 2, 3]}} expect(() => { manager.set({ service: { - port: 'invalid-port' as any - } + port: 'invalid-port' as any, + }, }); }).toThrow(ConfigValidationError); }); @@ -344,7 +335,7 @@ JSON_VALUE={"key": "value", "nested": {"array": [1, 2, 3]}} test('should handle loader priority conflicts', async () => { const configDir = join(TEST_DIR, 'config'); mkdirSync(configDir, { recursive: true }); - + writeFileSync( join(configDir, 'development.json'), JSON.stringify({ app: { name: 'file-config' } }) @@ -356,12 +347,12 @@ JSON_VALUE={"key": "value", "nested": {"array": [1, 2, 3]}} const manager = new ConfigManager({ loaders: [ new FileLoader(configDir, 'development'), // priority 50 - new EnvLoader('') // priority 100 - ] + new EnvLoader(''), // priority 100 + ], }); const config = await manager.initialize(appConfigSchema); - + // Environment should win due to higher priority expect(config.app.name).toBe('env-config'); }); @@ -369,16 +360,16 @@ JSON_VALUE={"key": "value", "nested": {"array": [1, 2, 3]}} test('should handle readonly environment variables', async () => { // Some system environment variables might be readonly const originalPath = process.env.PATH; - + // This should not cause the loader to fail const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); const config = await manager.initialize(); expect(config).toBeDefined(); - + // PATH should not be modified expect(process.env.PATH).toBe(originalPath); }); -}); \ No newline at end of file +}); diff --git a/libs/core/config/test/index.test.ts b/libs/core/config/test/index.test.ts index 215bb64..bc509f1 100644 --- a/libs/core/config/test/index.test.ts +++ b/libs/core/config/test/index.test.ts @@ -1,208 +1,202 @@ -import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; -import { writeFileSync, mkdirSync, rmSync } from 'fs'; -import { join } from 'path'; -import { - initializeConfig, - getConfig, - getConfigManager, - resetConfig, - getDatabaseConfig, - getServiceConfig, - getLoggingConfig, - getProviderConfig, - isDevelopment, - isProduction, - isTest, -} from '../src'; - -describe('Config Module', () => { - const testConfigDir = join(process.cwd(), 'test-config-module'); - const originalEnv = { ...process.env }; - - beforeEach(() => { - resetConfig(); - mkdirSync(testConfigDir, { recursive: true }); - - // Create test configuration files - const config = { - name: 'test-app', - version: '1.0.0', - service: { - name: 'test-service', - port: 3000, - }, - database: { - postgres: { - host: 'localhost', - port: 5432, - database: 'testdb', - user: 'testuser', - password: 'testpass', - }, - questdb: { - host: 'localhost', - httpPort: 9000, - pgPort: 8812, - }, - mongodb: { - host: 'localhost', - port: 27017, - database: 'testdb', - }, - dragonfly: { - host: 'localhost', - port: 6379, - }, - }, - logging: { - level: 'info', - format: 'json', - }, - providers: { - yahoo: { - enabled: true, - rateLimit: 5, - }, - qm: { - enabled: false, - apiKey: 'test-key', - }, - }, - environment: 'test', - }; - - writeFileSync( - join(testConfigDir, 'default.json'), - JSON.stringify(config, null, 2) - ); - }); - - afterEach(() => { - resetConfig(); - rmSync(testConfigDir, { recursive: true, force: true }); - process.env = { ...originalEnv }; - }); - - test('should initialize configuration', async () => { - const config = await initializeConfig(testConfigDir); - - expect(config.app.name).toBe('test-app'); - expect(config.service.port).toBe(3000); - expect(config.environment).toBe('test'); - }); - - test('should get configuration after initialization', async () => { - await initializeConfig(testConfigDir); - const config = getConfig(); - - expect(config.app.name).toBe('test-app'); - expect(config.database.postgres.host).toBe('localhost'); - }); - - test('should throw if getting config before initialization', () => { - expect(() => getConfig()).toThrow('Configuration not initialized'); - }); - - test('should get config manager instance', async () => { - await initializeConfig(testConfigDir); - const manager = getConfigManager(); - - expect(manager).toBeDefined(); - expect(manager.get().app.name).toBe('test-app'); - }); - - test('should get database configuration', async () => { - await initializeConfig(testConfigDir); - const dbConfig = getDatabaseConfig(); - - expect(dbConfig.postgres.host).toBe('localhost'); - expect(dbConfig.questdb.httpPort).toBe(9000); - expect(dbConfig.mongodb.database).toBe('testdb'); - }); - - test('should get service configuration', async () => { - await initializeConfig(testConfigDir); - const serviceConfig = getServiceConfig(); - - expect(serviceConfig.name).toBe('test-service'); - expect(serviceConfig.port).toBe(3000); - }); - - test('should get logging configuration', async () => { - await initializeConfig(testConfigDir); - const loggingConfig = getLoggingConfig(); - - expect(loggingConfig.level).toBe('info'); - expect(loggingConfig.format).toBe('json'); - }); - - test('should get provider configuration', async () => { - await initializeConfig(testConfigDir); - - const yahooConfig = getProviderConfig('yahoo'); - expect(yahooConfig.enabled).toBe(true); - expect(yahooConfig.rateLimit).toBe(5); - - const qmConfig = getProviderConfig('quoteMedia'); - expect(qmConfig.enabled).toBe(false); - expect(qmConfig.apiKey).toBe('test-key'); - }); - - test('should throw for non-existent provider', async () => { - await initializeConfig(testConfigDir); - - expect(() => getProviderConfig('nonexistent')).toThrow( - 'Provider configuration not found: nonexistent' - ); - }); - - test('should check environment correctly', async () => { - await initializeConfig(testConfigDir); - - expect(isTest()).toBe(true); - expect(isDevelopment()).toBe(false); - expect(isProduction()).toBe(false); - }); - - test('should handle environment overrides', async () => { - process.env.NODE_ENV = 'production'; - process.env.STOCKBOT_APP__NAME = 'env-override-app'; - process.env.STOCKBOT_DATABASE__POSTGRES__HOST = 'prod-db'; - - const prodConfig = { - database: { - postgres: { - host: 'prod-host', - port: 5432, - }, - }, - }; - - writeFileSync( - join(testConfigDir, 'production.json'), - JSON.stringify(prodConfig, null, 2) - ); - - const config = await initializeConfig(testConfigDir); - - expect(config.environment).toBe('production'); - expect(config.app.name).toBe('env-override-app'); - expect(config.database.postgres.host).toBe('prod-db'); - expect(isProduction()).toBe(true); - }); - - test('should reset configuration', async () => { - await initializeConfig(testConfigDir); - expect(() => getConfig()).not.toThrow(); - - resetConfig(); - expect(() => getConfig()).toThrow('Configuration not initialized'); - }); - - test('should maintain singleton instance', async () => { - const config1 = await initializeConfig(testConfigDir); - const config2 = await initializeConfig(testConfigDir); - - expect(config1).toBe(config2); - }); -}); \ No newline at end of file +import { mkdirSync, rmSync, writeFileSync } from 'fs'; +import { join } from 'path'; +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { + getConfig, + getConfigManager, + getDatabaseConfig, + getLoggingConfig, + getProviderConfig, + getServiceConfig, + initializeConfig, + isDevelopment, + isProduction, + isTest, + resetConfig, +} from '../src'; + +describe('Config Module', () => { + const testConfigDir = join(process.cwd(), 'test-config-module'); + const originalEnv = { ...process.env }; + + beforeEach(() => { + resetConfig(); + mkdirSync(testConfigDir, { recursive: true }); + + // Create test configuration files + const config = { + name: 'test-app', + version: '1.0.0', + service: { + name: 'test-service', + port: 3000, + }, + database: { + postgres: { + host: 'localhost', + port: 5432, + database: 'testdb', + user: 'testuser', + password: 'testpass', + }, + questdb: { + host: 'localhost', + httpPort: 9000, + pgPort: 8812, + }, + mongodb: { + host: 'localhost', + port: 27017, + database: 'testdb', + }, + dragonfly: { + host: 'localhost', + port: 6379, + }, + }, + logging: { + level: 'info', + format: 'json', + }, + providers: { + yahoo: { + enabled: true, + rateLimit: 5, + }, + qm: { + enabled: false, + apiKey: 'test-key', + }, + }, + environment: 'test', + }; + + writeFileSync(join(testConfigDir, 'default.json'), JSON.stringify(config, null, 2)); + }); + + afterEach(() => { + resetConfig(); + rmSync(testConfigDir, { recursive: true, force: true }); + process.env = { ...originalEnv }; + }); + + test('should initialize configuration', async () => { + const config = await initializeConfig(testConfigDir); + + expect(config.app.name).toBe('test-app'); + expect(config.service.port).toBe(3000); + expect(config.environment).toBe('test'); + }); + + test('should get configuration after initialization', async () => { + await initializeConfig(testConfigDir); + const config = getConfig(); + + expect(config.app.name).toBe('test-app'); + expect(config.database.postgres.host).toBe('localhost'); + }); + + test('should throw if getting config before initialization', () => { + expect(() => getConfig()).toThrow('Configuration not initialized'); + }); + + test('should get config manager instance', async () => { + await initializeConfig(testConfigDir); + const manager = getConfigManager(); + + expect(manager).toBeDefined(); + expect(manager.get().app.name).toBe('test-app'); + }); + + test('should get database configuration', async () => { + await initializeConfig(testConfigDir); + const dbConfig = getDatabaseConfig(); + + expect(dbConfig.postgres.host).toBe('localhost'); + expect(dbConfig.questdb.httpPort).toBe(9000); + expect(dbConfig.mongodb.database).toBe('testdb'); + }); + + test('should get service configuration', async () => { + await initializeConfig(testConfigDir); + const serviceConfig = getServiceConfig(); + + expect(serviceConfig.name).toBe('test-service'); + expect(serviceConfig.port).toBe(3000); + }); + + test('should get logging configuration', async () => { + await initializeConfig(testConfigDir); + const loggingConfig = getLoggingConfig(); + + expect(loggingConfig.level).toBe('info'); + expect(loggingConfig.format).toBe('json'); + }); + + test('should get provider configuration', async () => { + await initializeConfig(testConfigDir); + + const yahooConfig = getProviderConfig('yahoo'); + expect(yahooConfig.enabled).toBe(true); + expect(yahooConfig.rateLimit).toBe(5); + + const qmConfig = getProviderConfig('quoteMedia'); + expect(qmConfig.enabled).toBe(false); + expect(qmConfig.apiKey).toBe('test-key'); + }); + + test('should throw for non-existent provider', async () => { + await initializeConfig(testConfigDir); + + expect(() => getProviderConfig('nonexistent')).toThrow( + 'Provider configuration not found: nonexistent' + ); + }); + + test('should check environment correctly', async () => { + await initializeConfig(testConfigDir); + + expect(isTest()).toBe(true); + expect(isDevelopment()).toBe(false); + expect(isProduction()).toBe(false); + }); + + test('should handle environment overrides', async () => { + process.env.NODE_ENV = 'production'; + process.env.STOCKBOT_APP__NAME = 'env-override-app'; + process.env.STOCKBOT_DATABASE__POSTGRES__HOST = 'prod-db'; + + const prodConfig = { + database: { + postgres: { + host: 'prod-host', + port: 5432, + }, + }, + }; + + writeFileSync(join(testConfigDir, 'production.json'), JSON.stringify(prodConfig, null, 2)); + + const config = await initializeConfig(testConfigDir); + + expect(config.environment).toBe('production'); + expect(config.app.name).toBe('env-override-app'); + expect(config.database.postgres.host).toBe('prod-db'); + expect(isProduction()).toBe(true); + }); + + test('should reset configuration', async () => { + await initializeConfig(testConfigDir); + expect(() => getConfig()).not.toThrow(); + + resetConfig(); + expect(() => getConfig()).toThrow('Configuration not initialized'); + }); + + test('should maintain singleton instance', async () => { + const config1 = await initializeConfig(testConfigDir); + const config2 = await initializeConfig(testConfigDir); + + expect(config1).toBe(config2); + }); +}); diff --git a/libs/core/config/test/loaders.test.ts b/libs/core/config/test/loaders.test.ts index 40a484c..3f51003 100644 --- a/libs/core/config/test/loaders.test.ts +++ b/libs/core/config/test/loaders.test.ts @@ -1,181 +1,166 @@ -import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; -import { writeFileSync, mkdirSync, rmSync } from 'fs'; -import { join } from 'path'; -import { EnvLoader } from '../src/loaders/env.loader'; -import { FileLoader } from '../src/loaders/file.loader'; - -describe('EnvLoader', () => { - const originalEnv = { ...process.env }; - - afterEach(() => { - // Restore original environment - process.env = { ...originalEnv }; - }); - - test('should load environment variables with prefix', async () => { - process.env.TEST_APP_NAME = 'env-app'; - process.env.TEST_APP_VERSION = '1.0.0'; - process.env.TEST_DATABASE_HOST = 'env-host'; - process.env.TEST_DATABASE_PORT = '5432'; - process.env.OTHER_VAR = 'should-not-load'; - - const loader = new EnvLoader('TEST_', { convertCase: false, nestedDelimiter: null }); - const config = await loader.load(); - - expect(config.APP_NAME).toBe('env-app'); - expect(config.APP_VERSION).toBe('1.0.0'); - expect(config.DATABASE_HOST).toBe('env-host'); - expect(config.DATABASE_PORT).toBe(5432); // Should be parsed as number - expect(config.OTHER_VAR).toBeUndefined(); - }); - - test('should convert snake_case to camelCase', async () => { - process.env.TEST_DATABASE_CONNECTION_STRING = 'postgres://localhost'; - process.env.TEST_API_KEY_SECRET = 'secret123'; - - const loader = new EnvLoader('TEST_', { convertCase: true }); - const config = await loader.load(); - - expect(config.databaseConnectionString).toBe('postgres://localhost'); - expect(config.apiKeySecret).toBe('secret123'); - }); - - test('should parse JSON values', async () => { - process.env.TEST_SETTINGS = '{"feature": true, "limit": 100}'; - process.env.TEST_NUMBERS = '[1, 2, 3]'; - - const loader = new EnvLoader('TEST_', { parseJson: true }); - const config = await loader.load(); - - expect(config.SETTINGS).toEqual({ feature: true, limit: 100 }); - expect(config.NUMBERS).toEqual([1, 2, 3]); - }); - - test('should parse boolean and number values', async () => { - process.env.TEST_ENABLED = 'true'; - process.env.TEST_DISABLED = 'false'; - process.env.TEST_PORT = '3000'; - process.env.TEST_RATIO = '0.75'; - - const loader = new EnvLoader('TEST_', { parseValues: true }); - const config = await loader.load(); - - expect(config.ENABLED).toBe(true); - expect(config.DISABLED).toBe(false); - expect(config.PORT).toBe(3000); - expect(config.RATIO).toBe(0.75); - }); - - test('should handle nested object structure', async () => { - process.env.TEST_APP__NAME = 'nested-app'; - process.env.TEST_APP__SETTINGS__ENABLED = 'true'; - process.env.TEST_DATABASE__HOST = 'localhost'; - - const loader = new EnvLoader('TEST_', { - parseValues: true, - nestedDelimiter: '__' - }); - const config = await loader.load(); - - expect(config.APP).toEqual({ - NAME: 'nested-app', - SETTINGS: { - ENABLED: true - } - }); - expect(config.DATABASE).toEqual({ - HOST: 'localhost' - }); - }); -}); - -describe('FileLoader', () => { - const testDir = join(process.cwd(), 'test-config'); - - beforeEach(() => { - mkdirSync(testDir, { recursive: true }); - }); - - afterEach(() => { - rmSync(testDir, { recursive: true, force: true }); - }); - - test('should load JSON configuration file', async () => { - const config = { - app: { name: 'file-app', version: '1.0.0' }, - database: { host: 'localhost', port: 5432 } - }; - - writeFileSync( - join(testDir, 'default.json'), - JSON.stringify(config, null, 2) - ); - - const loader = new FileLoader(testDir); - const loaded = await loader.load(); - - expect(loaded).toEqual(config); - }); - - test('should load environment-specific configuration', async () => { - const defaultConfig = { - app: { name: 'app', port: 3000 }, - database: { host: 'localhost' } - }; - - const prodConfig = { - app: { port: 8080 }, - database: { host: 'prod-db' } - }; - - writeFileSync( - join(testDir, 'default.json'), - JSON.stringify(defaultConfig, null, 2) - ); - - writeFileSync( - join(testDir, 'production.json'), - JSON.stringify(prodConfig, null, 2) - ); - - const loader = new FileLoader(testDir, 'production'); - const loaded = await loader.load(); - - expect(loaded).toEqual({ - app: { name: 'app', port: 8080 }, - database: { host: 'prod-db' } - }); - }); - - test('should handle missing configuration files gracefully', async () => { - const loader = new FileLoader(testDir); - const loaded = await loader.load(); - - expect(loaded).toEqual({}); - }); - - test('should throw on invalid JSON', async () => { - writeFileSync( - join(testDir, 'default.json'), - 'invalid json content' - ); - - const loader = new FileLoader(testDir); - - await expect(loader.load()).rejects.toThrow(); - }); - - test('should support custom configuration', async () => { - const config = { custom: 'value' }; - - writeFileSync( - join(testDir, 'custom.json'), - JSON.stringify(config, null, 2) - ); - - const loader = new FileLoader(testDir); - const loaded = await loader.loadFile('custom.json'); - - expect(loaded).toEqual(config); - }); -}); \ No newline at end of file +import { mkdirSync, rmSync, writeFileSync } from 'fs'; +import { join } from 'path'; +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { EnvLoader } from '../src/loaders/env.loader'; +import { FileLoader } from '../src/loaders/file.loader'; + +describe('EnvLoader', () => { + const originalEnv = { ...process.env }; + + afterEach(() => { + // Restore original environment + process.env = { ...originalEnv }; + }); + + test('should load environment variables with prefix', async () => { + process.env.TEST_APP_NAME = 'env-app'; + process.env.TEST_APP_VERSION = '1.0.0'; + process.env.TEST_DATABASE_HOST = 'env-host'; + process.env.TEST_DATABASE_PORT = '5432'; + process.env.OTHER_VAR = 'should-not-load'; + + const loader = new EnvLoader('TEST_', { convertCase: false, nestedDelimiter: null }); + const config = await loader.load(); + + expect(config.APP_NAME).toBe('env-app'); + expect(config.APP_VERSION).toBe('1.0.0'); + expect(config.DATABASE_HOST).toBe('env-host'); + expect(config.DATABASE_PORT).toBe(5432); // Should be parsed as number + expect(config.OTHER_VAR).toBeUndefined(); + }); + + test('should convert snake_case to camelCase', async () => { + process.env.TEST_DATABASE_CONNECTION_STRING = 'postgres://localhost'; + process.env.TEST_API_KEY_SECRET = 'secret123'; + + const loader = new EnvLoader('TEST_', { convertCase: true }); + const config = await loader.load(); + + expect(config.databaseConnectionString).toBe('postgres://localhost'); + expect(config.apiKeySecret).toBe('secret123'); + }); + + test('should parse JSON values', async () => { + process.env.TEST_SETTINGS = '{"feature": true, "limit": 100}'; + process.env.TEST_NUMBERS = '[1, 2, 3]'; + + const loader = new EnvLoader('TEST_', { parseJson: true }); + const config = await loader.load(); + + expect(config.SETTINGS).toEqual({ feature: true, limit: 100 }); + expect(config.NUMBERS).toEqual([1, 2, 3]); + }); + + test('should parse boolean and number values', async () => { + process.env.TEST_ENABLED = 'true'; + process.env.TEST_DISABLED = 'false'; + process.env.TEST_PORT = '3000'; + process.env.TEST_RATIO = '0.75'; + + const loader = new EnvLoader('TEST_', { parseValues: true }); + const config = await loader.load(); + + expect(config.ENABLED).toBe(true); + expect(config.DISABLED).toBe(false); + expect(config.PORT).toBe(3000); + expect(config.RATIO).toBe(0.75); + }); + + test('should handle nested object structure', async () => { + process.env.TEST_APP__NAME = 'nested-app'; + process.env.TEST_APP__SETTINGS__ENABLED = 'true'; + process.env.TEST_DATABASE__HOST = 'localhost'; + + const loader = new EnvLoader('TEST_', { + parseValues: true, + nestedDelimiter: '__', + }); + const config = await loader.load(); + + expect(config.APP).toEqual({ + NAME: 'nested-app', + SETTINGS: { + ENABLED: true, + }, + }); + expect(config.DATABASE).toEqual({ + HOST: 'localhost', + }); + }); +}); + +describe('FileLoader', () => { + const testDir = join(process.cwd(), 'test-config'); + + beforeEach(() => { + mkdirSync(testDir, { recursive: true }); + }); + + afterEach(() => { + rmSync(testDir, { recursive: true, force: true }); + }); + + test('should load JSON configuration file', async () => { + const config = { + app: { name: 'file-app', version: '1.0.0' }, + database: { host: 'localhost', port: 5432 }, + }; + + writeFileSync(join(testDir, 'default.json'), JSON.stringify(config, null, 2)); + + const loader = new FileLoader(testDir); + const loaded = await loader.load(); + + expect(loaded).toEqual(config); + }); + + test('should load environment-specific configuration', async () => { + const defaultConfig = { + app: { name: 'app', port: 3000 }, + database: { host: 'localhost' }, + }; + + const prodConfig = { + app: { port: 8080 }, + database: { host: 'prod-db' }, + }; + + writeFileSync(join(testDir, 'default.json'), JSON.stringify(defaultConfig, null, 2)); + + writeFileSync(join(testDir, 'production.json'), JSON.stringify(prodConfig, null, 2)); + + const loader = new FileLoader(testDir, 'production'); + const loaded = await loader.load(); + + expect(loaded).toEqual({ + app: { name: 'app', port: 8080 }, + database: { host: 'prod-db' }, + }); + }); + + test('should handle missing configuration files gracefully', async () => { + const loader = new FileLoader(testDir); + const loaded = await loader.load(); + + expect(loaded).toEqual({}); + }); + + test('should throw on invalid JSON', async () => { + writeFileSync(join(testDir, 'default.json'), 'invalid json content'); + + const loader = new FileLoader(testDir); + + await expect(loader.load()).rejects.toThrow(); + }); + + test('should support custom configuration', async () => { + const config = { custom: 'value' }; + + writeFileSync(join(testDir, 'custom.json'), JSON.stringify(config, null, 2)); + + const loader = new FileLoader(testDir); + const loaded = await loader.loadFile('custom.json'); + + expect(loaded).toEqual(config); + }); +}); diff --git a/libs/core/config/test/provider-config.test.ts b/libs/core/config/test/provider-config.test.ts index 444aeec..0ed2365 100644 --- a/libs/core/config/test/provider-config.test.ts +++ b/libs/core/config/test/provider-config.test.ts @@ -1,11 +1,11 @@ -import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import { existsSync, mkdirSync, rmSync, writeFileSync } from 'fs'; +import { join } from 'path'; +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import { ConfigManager } from '../src/config-manager'; +import { getProviderConfig, resetConfig } from '../src/index'; import { EnvLoader } from '../src/loaders/env.loader'; import { FileLoader } from '../src/loaders/file.loader'; import { appConfigSchema } from '../src/schemas'; -import { resetConfig, getProviderConfig } from '../src/index'; -import { join } from 'path'; -import { mkdirSync, writeFileSync, rmSync, existsSync } from 'fs'; const TEST_DIR = join(__dirname, 'provider-tests'); @@ -15,10 +15,10 @@ describe('Provider Configuration Tests', () => { beforeEach(() => { // Save original environment originalEnv = { ...process.env }; - + // Reset config singleton resetConfig(); - + // Clean up test directory if (existsSync(TEST_DIR)) { rmSync(TEST_DIR, { recursive: true, force: true }); @@ -29,7 +29,7 @@ describe('Provider Configuration Tests', () => { afterEach(() => { // Restore original environment process.env = originalEnv; - + // Clean up resetConfig(); if (existsSync(TEST_DIR)) { @@ -44,7 +44,7 @@ describe('Provider Configuration Tests', () => { process.env.WEBSHARE_ENABLED = 'true'; const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); const config = await manager.initialize(appConfigSchema); @@ -64,7 +64,7 @@ describe('Provider Configuration Tests', () => { process.env.EOD_PRIORITY = '10'; const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); const config = await manager.initialize(appConfigSchema); @@ -88,7 +88,7 @@ describe('Provider Configuration Tests', () => { process.env.IB_PRIORITY = '5'; const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); const config = await manager.initialize(appConfigSchema); @@ -113,7 +113,7 @@ describe('Provider Configuration Tests', () => { process.env.QM_PRIORITY = '15'; const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); const config = await manager.initialize(appConfigSchema); @@ -136,7 +136,7 @@ describe('Provider Configuration Tests', () => { process.env.YAHOO_PRIORITY = '20'; const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); const config = await manager.initialize(appConfigSchema); @@ -153,27 +153,31 @@ describe('Provider Configuration Tests', () => { // Create a config file const configDir = join(TEST_DIR, 'config'); mkdirSync(configDir, { recursive: true }); - + writeFileSync( join(configDir, 'development.json'), - JSON.stringify({ - providers: { - eod: { - name: 'EOD Historical Data', - apiKey: 'file-eod-key', - baseUrl: 'https://file.eod.com/api', - tier: 'free', - enabled: false, - priority: 1 + JSON.stringify( + { + providers: { + eod: { + name: 'EOD Historical Data', + apiKey: 'file-eod-key', + baseUrl: 'https://file.eod.com/api', + tier: 'free', + enabled: false, + priority: 1, + }, + yahoo: { + name: 'Yahoo Finance', + baseUrl: 'https://file.yahoo.com', + enabled: true, + priority: 2, + }, }, - yahoo: { - name: 'Yahoo Finance', - baseUrl: 'https://file.yahoo.com', - enabled: true, - priority: 2 - } - } - }, null, 2) + }, + null, + 2 + ) ); // Set environment variables that should override file config @@ -183,10 +187,7 @@ describe('Provider Configuration Tests', () => { process.env.YAHOO_PRIORITY = '25'; const manager = new ConfigManager({ - loaders: [ - new FileLoader(configDir, 'development'), - new EnvLoader('') - ] + loaders: [new FileLoader(configDir, 'development'), new EnvLoader('')], }); const config = await manager.initialize(appConfigSchema); @@ -211,7 +212,7 @@ describe('Provider Configuration Tests', () => { process.env.IB_GATEWAY_PORT = 'not-a-number'; // Should be a number const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); // Should throw validation error @@ -226,7 +227,7 @@ describe('Provider Configuration Tests', () => { process.env.WEBSHARE_ENABLED = 'true'; const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); await manager.initialize(appConfigSchema); @@ -241,7 +242,9 @@ describe('Provider Configuration Tests', () => { expect((webshareConfig as any).apiKey).toBe('test-webshare-key'); // Test non-existent provider - expect(() => getProviderConfig('nonexistent')).toThrow('Provider configuration not found: nonexistent'); + expect(() => getProviderConfig('nonexistent')).toThrow( + 'Provider configuration not found: nonexistent' + ); }); test('should handle boolean string parsing correctly', async () => { @@ -253,7 +256,7 @@ describe('Provider Configuration Tests', () => { process.env.WEBSHARE_ENABLED = 'yes'; // Should be treated as string, not boolean const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); const config = await manager.initialize(appConfigSchema); @@ -272,7 +275,7 @@ describe('Provider Configuration Tests', () => { process.env.IB_GATEWAY_CLIENT_ID = '999'; const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); const config = await manager.initialize(appConfigSchema); @@ -300,9 +303,9 @@ YAHOO_BASE_URL=https://env-file.yahoo.com const originalCwd = process.cwd(); try { process.chdir(TEST_DIR); - + const manager = new ConfigManager({ - loaders: [new EnvLoader('')] + loaders: [new EnvLoader('')], }); const config = await manager.initialize(appConfigSchema); @@ -317,4 +320,4 @@ YAHOO_BASE_URL=https://env-file.yahoo.com process.chdir(originalCwd); } }); -}); \ No newline at end of file +}); diff --git a/libs/core/config/test/real-usage.test.ts b/libs/core/config/test/real-usage.test.ts index ff29453..e4aca62 100644 --- a/libs/core/config/test/real-usage.test.ts +++ b/libs/core/config/test/real-usage.test.ts @@ -1,6 +1,6 @@ -import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import { existsSync, mkdirSync, rmSync, writeFileSync } from 'fs'; import { join } from 'path'; +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import { getConfig, getDatabaseConfig, @@ -11,7 +11,7 @@ import { isDevelopment, isProduction, isTest, - resetConfig + resetConfig, } from '../src/index'; const TEST_DIR = join(__dirname, 'real-usage-tests'); @@ -23,13 +23,13 @@ describe('Real Usage Scenarios', () => { beforeEach(() => { originalEnv = { ...process.env }; originalCwd = process.cwd(); - + resetConfig(); - + if (existsSync(TEST_DIR)) { rmSync(TEST_DIR, { recursive: true, force: true }); } - + setupRealUsageScenarios(); }); @@ -37,7 +37,7 @@ describe('Real Usage Scenarios', () => { process.env = originalEnv; process.chdir(originalCwd); resetConfig(); - + if (existsSync(TEST_DIR)) { rmSync(TEST_DIR, { recursive: true, force: true }); } @@ -53,18 +53,18 @@ describe('Real Usage Scenarios', () => { // Test typical data-ingestion config access patterns expect(config.app.name).toBe('data-ingestion'); expect(config.service.port).toBe(3001); - + // Test database config access const dbConfig = getDatabaseConfig(); expect(dbConfig.postgres.host).toBe('localhost'); expect(dbConfig.postgres.port).toBe(5432); expect(dbConfig.questdb.host).toBe('localhost'); - + // Test provider access const yahooConfig = getProviderConfig('yahoo'); expect(yahooConfig).toBeDefined(); expect((yahooConfig as any).enabled).toBe(true); - + // Test environment helpers expect(isDevelopment()).toBe(true); expect(isProduction()).toBe(false); @@ -78,11 +78,11 @@ describe('Real Usage Scenarios', () => { expect(config.app.name).toBe('web-api'); expect(config.service.port).toBe(4000); - + // Web API should have access to all the same configs const serviceConfig = getServiceConfig(); expect(serviceConfig.name).toBe('web-api'); - + const loggingConfig = getLoggingConfig(); expect(loggingConfig.level).toBe('info'); }); @@ -96,7 +96,7 @@ describe('Real Usage Scenarios', () => { // Libraries should inherit from root config expect(config.app.name).toBe('cache-lib'); expect(config.app.version).toBe('1.0.0'); // From root - + // Should have access to cache config const dbConfig = getDatabaseConfig(); expect(dbConfig.dragonfly).toBeDefined(); @@ -106,7 +106,7 @@ describe('Real Usage Scenarios', () => { test('should handle production environment correctly', async () => { process.env.NODE_ENV = 'production'; - + const dataServiceDir = join(TEST_DIR, 'apps', 'data-ingestion'); process.chdir(dataServiceDir); @@ -115,14 +115,14 @@ describe('Real Usage Scenarios', () => { expect(config.environment).toBe('production'); expect(config.logging.level).toBe('warn'); // Production should use different log level - + expect(isProduction()).toBe(true); expect(isDevelopment()).toBe(false); }); test('should handle test environment correctly', async () => { process.env.NODE_ENV = 'test'; - + const dataServiceDir = join(TEST_DIR, 'apps', 'data-ingestion'); process.chdir(dataServiceDir); @@ -131,7 +131,7 @@ describe('Real Usage Scenarios', () => { expect(config.environment).toBe('test'); expect(config.logging.level).toBe('debug'); // Test should use debug level - + expect(isTest()).toBe(true); expect(isDevelopment()).toBe(false); }); @@ -153,10 +153,10 @@ describe('Real Usage Scenarios', () => { const dbConfig = getDatabaseConfig(); expect(dbConfig.postgres.host).toBe('prod-db.example.com'); expect(dbConfig.postgres.port).toBe(5433); - + const serviceConfig = getServiceConfig(); expect(serviceConfig.port).toBe(8080); - + const eodConfig = getProviderConfig('eod'); expect((eodConfig as any).apiKey).toBe('prod-eod-key'); }); @@ -168,8 +168,10 @@ describe('Real Usage Scenarios', () => { const config = await initializeServiceConfig(); // Should throw for non-existent providers - expect(() => getProviderConfig('nonexistent')).toThrow('Provider configuration not found: nonexistent'); - + expect(() => getProviderConfig('nonexistent')).toThrow( + 'Provider configuration not found: nonexistent' + ); + // Should work for providers that exist but might not be configured // (they should have defaults from schema) const yahooConfig = getProviderConfig('yahoo'); @@ -181,18 +183,18 @@ describe('Real Usage Scenarios', () => { process.chdir(dataServiceDir); const config = await initializeServiceConfig(); - + // Test various access patterns used in real applications const configManager = (await import('../src/index')).getConfigManager(); - + // Direct path access expect(configManager.getValue('app.name')).toBe('data-ingestion'); expect(configManager.getValue('service.port')).toBe(3001); - + // Check if paths exist expect(configManager.has('app.name')).toBe(true); expect(configManager.has('nonexistent.path')).toBe(false); - + // Typed access const port = configManager.getValue('service.port'); expect(typeof port).toBe('number'); @@ -205,39 +207,39 @@ describe('Real Usage Scenarios', () => { await initializeServiceConfig(); const configManager = (await import('../src/index')).getConfigManager(); - + // Update config at runtime (useful for testing) configManager.set({ service: { - port: 9999 - } + port: 9999, + }, }); - + const updatedConfig = getConfig(); expect(updatedConfig.service.port).toBe(9999); - + // Other values should be preserved expect(updatedConfig.app.name).toBe('data-ingestion'); }); test('should work across multiple service initializations', async () => { // Simulate multiple services in the same process (like tests) - + // First service const dataServiceDir = join(TEST_DIR, 'apps', 'data-ingestion'); process.chdir(dataServiceDir); - + let config = await initializeServiceConfig(); expect(config.app.name).toBe('data-ingestion'); - - // Reset and switch to another service + + // Reset and switch to another service resetConfig(); const webApiDir = join(TEST_DIR, 'apps', 'web-api'); process.chdir(webApiDir); - + config = await initializeServiceConfig(); expect(config.app.name).toBe('web-api'); - + // Each service should get its own config expect(config.service.port).toBe(4000); // web-api port }); @@ -263,7 +265,7 @@ function setupRealUsageScenarios() { development: { app: { name: 'stock-bot-monorepo', - version: '1.0.0' + version: '1.0.0', }, database: { postgres: { @@ -271,116 +273,125 @@ function setupRealUsageScenarios() { port: 5432, database: 'trading_bot', username: 'trading_user', - password: 'trading_pass_dev' + password: 'trading_pass_dev', }, questdb: { host: 'localhost', port: 9009, - database: 'questdb' + database: 'questdb', }, mongodb: { host: 'localhost', port: 27017, - database: 'stock' + database: 'stock', }, dragonfly: { host: 'localhost', - port: 6379 - } + port: 6379, + }, }, logging: { level: 'info', - format: 'json' + format: 'json', }, providers: { yahoo: { name: 'Yahoo Finance', enabled: true, priority: 1, - baseUrl: 'https://query1.finance.yahoo.com' + baseUrl: 'https://query1.finance.yahoo.com', }, eod: { name: 'EOD Historical Data', enabled: false, priority: 2, apiKey: 'demo-api-key', - baseUrl: 'https://eodhistoricaldata.com/api' - } - } + baseUrl: 'https://eodhistoricaldata.com/api', + }, + }, }, production: { logging: { - level: 'warn' + level: 'warn', }, database: { postgres: { host: 'prod-postgres.internal', - port: 5432 - } - } + port: 5432, + }, + }, }, test: { logging: { - level: 'debug' + level: 'debug', }, database: { postgres: { - database: 'trading_bot_test' - } - } - } + database: 'trading_bot_test', + }, + }, + }, }; Object.entries(rootConfigs).forEach(([env, config]) => { - writeFileSync( - join(scenarios.root, 'config', `${env}.json`), - JSON.stringify(config, null, 2) - ); + writeFileSync(join(scenarios.root, 'config', `${env}.json`), JSON.stringify(config, null, 2)); }); // Data service config writeFileSync( join(scenarios.dataService, 'config', 'development.json'), - JSON.stringify({ - app: { - name: 'data-ingestion' + JSON.stringify( + { + app: { + name: 'data-ingestion', + }, + service: { + name: 'data-ingestion', + port: 3001, + workers: 2, + }, }, - service: { - name: 'data-ingestion', - port: 3001, - workers: 2 - } - }, null, 2) + null, + 2 + ) ); // Web API config writeFileSync( join(scenarios.webApi, 'config', 'development.json'), - JSON.stringify({ - app: { - name: 'web-api' + JSON.stringify( + { + app: { + name: 'web-api', + }, + service: { + name: 'web-api', + port: 4000, + cors: { + origin: ['http://localhost:3000', 'http://localhost:4200'], + }, + }, }, - service: { - name: 'web-api', - port: 4000, - cors: { - origin: ['http://localhost:3000', 'http://localhost:4200'] - } - } - }, null, 2) + null, + 2 + ) ); // Cache lib config writeFileSync( join(scenarios.cacheLib, 'config', 'development.json'), - JSON.stringify({ - app: { - name: 'cache-lib' + JSON.stringify( + { + app: { + name: 'cache-lib', + }, + service: { + name: 'cache-lib', + }, }, - service: { - name: 'cache-lib' - } - }, null, 2) + null, + 2 + ) ); // Root .env file @@ -401,4 +412,4 @@ WEBSHARE_API_KEY=demo-webshare-key DATA_SERVICE_RATE_LIMIT=1000 ` ); -} \ No newline at end of file +} diff --git a/libs/core/config/tsconfig.json b/libs/core/config/tsconfig.json index 3f476c8..9405533 100644 --- a/libs/core/config/tsconfig.json +++ b/libs/core/config/tsconfig.json @@ -6,6 +6,5 @@ "composite": true }, "include": ["src/**/*"], - "references": [ - ] -} \ No newline at end of file + "references": [] +} diff --git a/libs/core/di/package.json b/libs/core/di/package.json index 1338e03..5dda6f4 100644 --- a/libs/core/di/package.json +++ b/libs/core/di/package.json @@ -1,17 +1,17 @@ -{ - "name": "@stock-bot/di", - "version": "1.0.0", - "main": "./src/index.ts", - "types": "./src/index.ts", - "scripts": { - "build": "tsc", - "clean": "rm -rf dist" - }, - "dependencies": { - "@stock-bot/config": "workspace:*", - "@stock-bot/logger": "workspace:*" - }, - "devDependencies": { - "@types/pg": "^8.10.7" - } -} \ No newline at end of file +{ + "name": "@stock-bot/di", + "version": "1.0.0", + "main": "./src/index.ts", + "types": "./src/index.ts", + "scripts": { + "build": "tsc", + "clean": "rm -rf dist" + }, + "dependencies": { + "@stock-bot/config": "workspace:*", + "@stock-bot/logger": "workspace:*" + }, + "devDependencies": { + "@types/pg": "^8.10.7" + } +} diff --git a/libs/core/di/src/awilix-container.ts b/libs/core/di/src/awilix-container.ts index a482518..4362959 100644 --- a/libs/core/di/src/awilix-container.ts +++ b/libs/core/di/src/awilix-container.ts @@ -1,294 +1,301 @@ -/** - * Awilix DI Container Setup - * Creates a decoupled, reusable dependency injection container - */ - -import { Browser } from '@stock-bot/browser'; -import { createCache, type CacheProvider } from '@stock-bot/cache'; -import type { IServiceContainer } from '@stock-bot/handlers'; -import { getLogger, type Logger } from '@stock-bot/logger'; -import { MongoDBClient } from '@stock-bot/mongodb'; -import { PostgreSQLClient } from '@stock-bot/postgres'; -import { ProxyManager } from '@stock-bot/proxy'; -import { QuestDBClient } from '@stock-bot/questdb'; -import { type QueueManager } from '@stock-bot/queue'; -import { asFunction, asValue, createContainer, InjectionMode, type AwilixContainer } from 'awilix'; -import { z } from 'zod'; - -// Configuration schema with validation -const appConfigSchema = z.object({ - redis: z.object({ - enabled: z.boolean().optional(), - host: z.string(), - port: z.number(), - password: z.string().optional(), - username: z.string().optional(), - db: z.number().optional(), - }), - mongodb: z.object({ - enabled: z.boolean().optional(), - uri: z.string(), - database: z.string(), - }), - postgres: z.object({ - enabled: z.boolean().optional(), - host: z.string(), - port: z.number(), - database: z.string(), - user: z.string(), - password: z.string(), - }), - questdb: z.object({ - enabled: z.boolean().optional(), - host: z.string(), - httpPort: z.number().optional(), - pgPort: z.number().optional(), - influxPort: z.number().optional(), - database: z.string().optional(), - }).optional(), - proxy: z.object({ - cachePrefix: z.string().optional(), - ttl: z.number().optional(), - }).optional(), - browser: z.object({ - headless: z.boolean().optional(), - timeout: z.number().optional(), - }).optional(), -}); - -export type AppConfig = z.infer; - -/** - * Service type definitions for type-safe resolution - */ -export interface ServiceDefinitions { - // Configuration - config: AppConfig; - logger: Logger; - - // Core services - cache: CacheProvider | null; - proxyManager: ProxyManager | null; - browser: Browser; - queueManager: QueueManager | null; - - // Database clients - mongoClient: MongoDBClient | null; - postgresClient: PostgreSQLClient | null; - questdbClient: QuestDBClient | null; - - // Aggregate service container - serviceContainer: IServiceContainer; -} - -/** - * Create and configure the DI container with type safety - */ -export function createServiceContainer(rawConfig: unknown): AwilixContainer { - // Validate configuration - const config = appConfigSchema.parse(rawConfig); - - const container = createContainer({ - injectionMode: InjectionMode.PROXY, - }); - - // Register configuration values - const registrations: any = { - // Configuration - config: asValue(config), - redisConfig: asValue(config.redis), - mongoConfig: asValue(config.mongodb), - postgresConfig: asValue(config.postgres), - questdbConfig: asValue(config.questdb || { host: 'localhost', httpPort: 9000, pgPort: 8812, influxPort: 9009 }), - - // Core services with dependency injection - logger: asFunction(() => getLogger('app')).singleton(), - }; - - // Conditionally register cache/dragonfly - if (config.redis?.enabled !== false) { - registrations.cache = asFunction(({ redisConfig, logger }) => - createCache({ - redisConfig, - logger, - keyPrefix: 'cache:', - ttl: 3600, - enableMetrics: true, - }) - ).singleton(); - } else { - registrations.cache = asValue(null); - } - - // Proxy manager depends on cache - registrations.proxyManager = asFunction(({ cache, config, logger }) => { - if (!cache) { - logger.warn('Cache is disabled, ProxyManager will have limited functionality'); - return null; - } - const manager = new ProxyManager( - cache, - config.proxy || {}, - logger - ); - return manager; - }).singleton(); - - // Conditionally register MongoDB client - if (config.mongodb?.enabled !== false) { - registrations.mongoClient = asFunction(({ mongoConfig, logger }) => { - return new MongoDBClient(mongoConfig, logger); - }).singleton(); - } else { - registrations.mongoClient = asValue(null); - } - - // Conditionally register PostgreSQL client - if (config.postgres?.enabled !== false) { - registrations.postgresClient = asFunction(({ postgresConfig, logger }) => { - return new PostgreSQLClient( - { - host: postgresConfig.host, - port: postgresConfig.port, - database: postgresConfig.database, - username: postgresConfig.user, - password: postgresConfig.password, - }, - logger - ); - }).singleton(); - } else { - registrations.postgresClient = asValue(null); - } - - // Conditionally register QuestDB client - if (config.questdb?.enabled !== false) { - registrations.questdbClient = asFunction(({ questdbConfig, logger }) => { - console.log('Creating QuestDB client with config:', questdbConfig); - return new QuestDBClient( - { - host: questdbConfig.host, - httpPort: questdbConfig.httpPort, - pgPort: questdbConfig.pgPort, - influxPort: questdbConfig.influxPort, - database: questdbConfig.database, - // QuestDB appears to require default credentials - user: 'admin', - password: 'quest', - }, - logger - ); - }).singleton(); - } else { - registrations.questdbClient = asValue(null); - } - - // Queue manager - placeholder until decoupled from singleton - registrations.queueManager = asFunction(({ redisConfig, cache, logger }) => { - // Import dynamically to avoid circular dependency - const { QueueManager } = require('@stock-bot/queue'); - - // Check if already initialized (singleton pattern) - if (QueueManager.isInitialized()) { - return QueueManager.getInstance(); - } - - // Initialize if not already done - return QueueManager.initialize({ - redis: { host: redisConfig.host, port: redisConfig.port, db: redisConfig.db }, - enableScheduledJobs: true, - delayWorkerStart: true // We'll start workers manually - }); - }).singleton(); - - // Browser automation - registrations.browser = asFunction(({ config, logger }) => { - return new Browser(logger, config.browser); - }).singleton(); - - // Build the IServiceContainer for handlers - registrations.serviceContainer = asFunction((cradle) => ({ - logger: cradle.logger, - cache: cradle.cache, - proxy: cradle.proxyManager, - browser: cradle.browser, - mongodb: cradle.mongoClient, - postgres: cradle.postgresClient, - questdb: cradle.questdbClient, - queue: cradle.queueManager, - } as IServiceContainer)).singleton(); - - container.register(registrations); - return container; -} - -/** - * Initialize async services after container creation - */ -export async function initializeServices(container: AwilixContainer): Promise { - const logger = container.resolve('logger'); - const config = container.resolve('config'); - - try { - // Wait for cache to be ready first (if enabled) - const cache = container.resolve('cache'); - if (cache && typeof cache.waitForReady === 'function') { - await cache.waitForReady(10000); - logger.info('Cache is ready'); - } else if (config.redis?.enabled === false) { - logger.info('Cache is disabled'); - } - - // Initialize proxy manager (depends on cache) - const proxyManager = container.resolve('proxyManager'); - if (proxyManager && typeof proxyManager.initialize === 'function') { - await proxyManager.initialize(); - logger.info('Proxy manager initialized'); - } else { - logger.info('Proxy manager is disabled (requires cache)'); - } - - // Connect MongoDB client (if enabled) - const mongoClient = container.resolve('mongoClient'); - if (mongoClient && typeof mongoClient.connect === 'function') { - await mongoClient.connect(); - logger.info('MongoDB connected'); - } else if (config.mongodb?.enabled === false) { - logger.info('MongoDB is disabled'); - } - - // Connect PostgreSQL client (if enabled) - const postgresClient = container.resolve('postgresClient'); - if (postgresClient && typeof postgresClient.connect === 'function') { - await postgresClient.connect(); - logger.info('PostgreSQL connected'); - } else if (config.postgres?.enabled === false) { - logger.info('PostgreSQL is disabled'); - } - - // Connect QuestDB client (if enabled) - const questdbClient = container.resolve('questdbClient'); - if (questdbClient && typeof questdbClient.connect === 'function') { - await questdbClient.connect(); - logger.info('QuestDB connected'); - } else if (config.questdb?.enabled === false) { - logger.info('QuestDB is disabled'); - } - - // Initialize browser if configured - const browser = container.resolve('browser'); - if (browser && typeof browser.initialize === 'function') { - await browser.initialize(); - logger.info('Browser initialized'); - } - - logger.info('All services initialized successfully'); - } catch (error) { - logger.error('Failed to initialize services', { error }); - throw error; - } -} - -// Export typed container -export type ServiceContainer = AwilixContainer; -export type ServiceCradle = ServiceDefinitions; \ No newline at end of file +/** + * Awilix DI Container Setup + * Creates a decoupled, reusable dependency injection container + */ + +import { asFunction, asValue, createContainer, InjectionMode, type AwilixContainer } from 'awilix'; +import { z } from 'zod'; +import { Browser } from '@stock-bot/browser'; +import { createCache, type CacheProvider } from '@stock-bot/cache'; +import type { IServiceContainer } from '@stock-bot/handlers'; +import { getLogger, type Logger } from '@stock-bot/logger'; +import { MongoDBClient } from '@stock-bot/mongodb'; +import { PostgreSQLClient } from '@stock-bot/postgres'; +import { ProxyManager } from '@stock-bot/proxy'; +import { QuestDBClient } from '@stock-bot/questdb'; +import { type QueueManager } from '@stock-bot/queue'; + +// Configuration schema with validation +const appConfigSchema = z.object({ + redis: z.object({ + enabled: z.boolean().optional(), + host: z.string(), + port: z.number(), + password: z.string().optional(), + username: z.string().optional(), + db: z.number().optional(), + }), + mongodb: z.object({ + enabled: z.boolean().optional(), + uri: z.string(), + database: z.string(), + }), + postgres: z.object({ + enabled: z.boolean().optional(), + host: z.string(), + port: z.number(), + database: z.string(), + user: z.string(), + password: z.string(), + }), + questdb: z + .object({ + enabled: z.boolean().optional(), + host: z.string(), + httpPort: z.number().optional(), + pgPort: z.number().optional(), + influxPort: z.number().optional(), + database: z.string().optional(), + }) + .optional(), + proxy: z + .object({ + cachePrefix: z.string().optional(), + ttl: z.number().optional(), + }) + .optional(), + browser: z + .object({ + headless: z.boolean().optional(), + timeout: z.number().optional(), + }) + .optional(), +}); + +export type AppConfig = z.infer; + +/** + * Service type definitions for type-safe resolution + */ +export interface ServiceDefinitions { + // Configuration + config: AppConfig; + logger: Logger; + + // Core services + cache: CacheProvider | null; + proxyManager: ProxyManager | null; + browser: Browser; + queueManager: QueueManager | null; + + // Database clients + mongoClient: MongoDBClient | null; + postgresClient: PostgreSQLClient | null; + questdbClient: QuestDBClient | null; + + // Aggregate service container + serviceContainer: IServiceContainer; +} + +/** + * Create and configure the DI container with type safety + */ +export function createServiceContainer(rawConfig: unknown): AwilixContainer { + // Validate configuration + const config = appConfigSchema.parse(rawConfig); + + const container = createContainer({ + injectionMode: InjectionMode.PROXY, + }); + + // Register configuration values + const registrations: any = { + // Configuration + config: asValue(config), + redisConfig: asValue(config.redis), + mongoConfig: asValue(config.mongodb), + postgresConfig: asValue(config.postgres), + questdbConfig: asValue( + config.questdb || { host: 'localhost', httpPort: 9000, pgPort: 8812, influxPort: 9009 } + ), + + // Core services with dependency injection + logger: asFunction(() => getLogger('app')).singleton(), + }; + + // Conditionally register cache/dragonfly + if (config.redis?.enabled !== false) { + registrations.cache = asFunction(({ redisConfig, logger }) => + createCache({ + redisConfig, + logger, + keyPrefix: 'cache:', + ttl: 3600, + enableMetrics: true, + }) + ).singleton(); + } else { + registrations.cache = asValue(null); + } + + // Proxy manager depends on cache + registrations.proxyManager = asFunction(({ cache, config, logger }) => { + if (!cache) { + logger.warn('Cache is disabled, ProxyManager will have limited functionality'); + return null; + } + const manager = new ProxyManager(cache, config.proxy || {}, logger); + return manager; + }).singleton(); + + // Conditionally register MongoDB client + if (config.mongodb?.enabled !== false) { + registrations.mongoClient = asFunction(({ mongoConfig, logger }) => { + return new MongoDBClient(mongoConfig, logger); + }).singleton(); + } else { + registrations.mongoClient = asValue(null); + } + + // Conditionally register PostgreSQL client + if (config.postgres?.enabled !== false) { + registrations.postgresClient = asFunction(({ postgresConfig, logger }) => { + return new PostgreSQLClient( + { + host: postgresConfig.host, + port: postgresConfig.port, + database: postgresConfig.database, + username: postgresConfig.user, + password: postgresConfig.password, + }, + logger + ); + }).singleton(); + } else { + registrations.postgresClient = asValue(null); + } + + // Conditionally register QuestDB client + if (config.questdb?.enabled !== false) { + registrations.questdbClient = asFunction(({ questdbConfig, logger }) => { + console.log('Creating QuestDB client with config:', questdbConfig); + return new QuestDBClient( + { + host: questdbConfig.host, + httpPort: questdbConfig.httpPort, + pgPort: questdbConfig.pgPort, + influxPort: questdbConfig.influxPort, + database: questdbConfig.database, + // QuestDB appears to require default credentials + user: 'admin', + password: 'quest', + }, + logger + ); + }).singleton(); + } else { + registrations.questdbClient = asValue(null); + } + + // Queue manager - placeholder until decoupled from singleton + registrations.queueManager = asFunction(({ redisConfig, cache, logger }) => { + // Import dynamically to avoid circular dependency + const { QueueManager } = require('@stock-bot/queue'); + + // Check if already initialized (singleton pattern) + if (QueueManager.isInitialized()) { + return QueueManager.getInstance(); + } + + // Initialize if not already done + return QueueManager.initialize({ + redis: { host: redisConfig.host, port: redisConfig.port, db: redisConfig.db }, + enableScheduledJobs: true, + delayWorkerStart: true, // We'll start workers manually + }); + }).singleton(); + + // Browser automation + registrations.browser = asFunction(({ config, logger }) => { + return new Browser(logger, config.browser); + }).singleton(); + + // Build the IServiceContainer for handlers + registrations.serviceContainer = asFunction( + cradle => + ({ + logger: cradle.logger, + cache: cradle.cache, + proxy: cradle.proxyManager, + browser: cradle.browser, + mongodb: cradle.mongoClient, + postgres: cradle.postgresClient, + questdb: cradle.questdbClient, + queue: cradle.queueManager, + }) as IServiceContainer + ).singleton(); + + container.register(registrations); + return container; +} + +/** + * Initialize async services after container creation + */ +export async function initializeServices(container: AwilixContainer): Promise { + const logger = container.resolve('logger'); + const config = container.resolve('config'); + + try { + // Wait for cache to be ready first (if enabled) + const cache = container.resolve('cache'); + if (cache && typeof cache.waitForReady === 'function') { + await cache.waitForReady(10000); + logger.info('Cache is ready'); + } else if (config.redis?.enabled === false) { + logger.info('Cache is disabled'); + } + + // Initialize proxy manager (depends on cache) + const proxyManager = container.resolve('proxyManager'); + if (proxyManager && typeof proxyManager.initialize === 'function') { + await proxyManager.initialize(); + logger.info('Proxy manager initialized'); + } else { + logger.info('Proxy manager is disabled (requires cache)'); + } + + // Connect MongoDB client (if enabled) + const mongoClient = container.resolve('mongoClient'); + if (mongoClient && typeof mongoClient.connect === 'function') { + await mongoClient.connect(); + logger.info('MongoDB connected'); + } else if (config.mongodb?.enabled === false) { + logger.info('MongoDB is disabled'); + } + + // Connect PostgreSQL client (if enabled) + const postgresClient = container.resolve('postgresClient'); + if (postgresClient && typeof postgresClient.connect === 'function') { + await postgresClient.connect(); + logger.info('PostgreSQL connected'); + } else if (config.postgres?.enabled === false) { + logger.info('PostgreSQL is disabled'); + } + + // Connect QuestDB client (if enabled) + const questdbClient = container.resolve('questdbClient'); + if (questdbClient && typeof questdbClient.connect === 'function') { + await questdbClient.connect(); + logger.info('QuestDB connected'); + } else if (config.questdb?.enabled === false) { + logger.info('QuestDB is disabled'); + } + + // Initialize browser if configured + const browser = container.resolve('browser'); + if (browser && typeof browser.initialize === 'function') { + await browser.initialize(); + logger.info('Browser initialized'); + } + + logger.info('All services initialized successfully'); + } catch (error) { + logger.error('Failed to initialize services', { error }); + throw error; + } +} + +// Export typed container +export type ServiceContainer = AwilixContainer; +export type ServiceCradle = ServiceDefinitions; diff --git a/libs/core/di/src/index.ts b/libs/core/di/src/index.ts index 4964ba6..4acf13d 100644 --- a/libs/core/di/src/index.ts +++ b/libs/core/di/src/index.ts @@ -1,13 +1,13 @@ -// Export all dependency injection components -export * from './operation-context'; -export * from './pool-size-calculator'; -export * from './types'; - -// Awilix container exports -export { - createServiceContainer, - initializeServices, - type AppConfig, - type ServiceCradle, - type ServiceContainer -} from './awilix-container'; \ No newline at end of file +// Export all dependency injection components +export * from './operation-context'; +export * from './pool-size-calculator'; +export * from './types'; + +// Awilix container exports +export { + createServiceContainer, + initializeServices, + type AppConfig, + type ServiceCradle, + type ServiceContainer, +} from './awilix-container'; diff --git a/libs/core/di/src/operation-context.ts b/libs/core/di/src/operation-context.ts index 796abc7..6aba882 100644 --- a/libs/core/di/src/operation-context.ts +++ b/libs/core/di/src/operation-context.ts @@ -3,6 +3,7 @@ */ import { getLogger, type Logger } from '@stock-bot/logger'; + interface ServiceResolver { resolve(serviceName: string): T; resolveAsync(serviceName: string): Promise; @@ -23,17 +24,19 @@ export class OperationContext { public readonly metadata: Record; private readonly container?: ServiceResolver; private readonly startTime: Date; - + constructor(options: OperationContextOptions) { this.container = options.container; this.metadata = options.metadata || {}; this.traceId = options.traceId || this.generateTraceId(); this.startTime = new Date(); - - this.logger = options.parentLogger || getLogger(`${options.handlerName}:${options.operationName}`, { - traceId: this.traceId, - metadata: this.metadata, - }); + + this.logger = + options.parentLogger || + getLogger(`${options.handlerName}:${options.operationName}`, { + traceId: this.traceId, + metadata: this.metadata, + }); } /** @@ -42,8 +45,8 @@ export class OperationContext { static create( handlerName: string, operationName: string, - options: { - container?: ServiceResolver; + options: { + container?: ServiceResolver; parentLogger?: Logger; metadata?: Record; traceId?: string; @@ -95,7 +98,7 @@ export class OperationContext { */ logCompletion(success: boolean, error?: Error): void { const executionTime = this.getExecutionTime(); - + if (success) { this.logger.info('Operation completed successfully', { executionTime, @@ -138,4 +141,4 @@ export class OperationContext { private generateTraceId(): string { return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; } -} \ No newline at end of file +} diff --git a/libs/core/di/src/pool-size-calculator.ts b/libs/core/di/src/pool-size-calculator.ts index cd48b1f..53654e2 100644 --- a/libs/core/di/src/pool-size-calculator.ts +++ b/libs/core/di/src/pool-size-calculator.ts @@ -1,80 +1,82 @@ -import type { ConnectionPoolConfig } from './types'; - -export interface PoolSizeRecommendation { - min: number; - max: number; - idle: number; -} - -export class PoolSizeCalculator { - private static readonly DEFAULT_SIZES: Record = { - // Service-level defaults - 'data-ingestion': { min: 5, max: 50, idle: 10 }, - 'data-pipeline': { min: 3, max: 30, idle: 5 }, - 'processing-service': { min: 2, max: 20, idle: 3 }, - 'web-api': { min: 2, max: 10, idle: 2 }, - 'portfolio-service': { min: 2, max: 15, idle: 3 }, - 'strategy-service': { min: 3, max: 25, idle: 5 }, - 'execution-service': { min: 2, max: 10, idle: 2 }, - - // Handler-level defaults - 'batch-import': { min: 10, max: 100, idle: 20 }, - 'real-time': { min: 2, max: 10, idle: 3 }, - 'analytics': { min: 5, max: 30, idle: 10 }, - 'reporting': { min: 3, max: 20, idle: 5 }, - }; - - static calculate( - serviceName: string, - handlerName?: string, - customConfig?: Partial - ): PoolSizeRecommendation { - // Check for custom configuration first - if (customConfig?.minConnections && customConfig?.maxConnections) { - return { - min: customConfig.minConnections, - max: customConfig.maxConnections, - idle: Math.floor((customConfig.minConnections + customConfig.maxConnections) / 4), - }; - } - - // Try handler-specific sizes first, then service-level - const key = handlerName || serviceName; - const recommendation = this.DEFAULT_SIZES[key] || this.DEFAULT_SIZES[serviceName]; - - if (recommendation) { - return { ...recommendation }; - } - - // Fall back to generic defaults - return { - min: 2, - max: 10, - idle: 3, - }; - } - - static getOptimalPoolSize( - expectedConcurrency: number, - averageQueryTimeMs: number, - targetLatencyMs: number - ): number { - // Little's Law: L = λ * W - // L = number of connections needed - // λ = arrival rate (requests per second) - // W = average time in system (seconds) - - const requestsPerSecond = expectedConcurrency; - const averageTimeInSystem = averageQueryTimeMs / 1000; - - const minConnections = Math.ceil(requestsPerSecond * averageTimeInSystem); - - // Add buffer for burst traffic (20% overhead) - const recommendedSize = Math.ceil(minConnections * 1.2); - - // Ensure we meet target latency - const latencyBasedSize = Math.ceil(expectedConcurrency * (averageQueryTimeMs / targetLatencyMs)); - - return Math.max(recommendedSize, latencyBasedSize, 2); // Minimum 2 connections - } -} \ No newline at end of file +import type { ConnectionPoolConfig } from './types'; + +export interface PoolSizeRecommendation { + min: number; + max: number; + idle: number; +} + +export class PoolSizeCalculator { + private static readonly DEFAULT_SIZES: Record = { + // Service-level defaults + 'data-ingestion': { min: 5, max: 50, idle: 10 }, + 'data-pipeline': { min: 3, max: 30, idle: 5 }, + 'processing-service': { min: 2, max: 20, idle: 3 }, + 'web-api': { min: 2, max: 10, idle: 2 }, + 'portfolio-service': { min: 2, max: 15, idle: 3 }, + 'strategy-service': { min: 3, max: 25, idle: 5 }, + 'execution-service': { min: 2, max: 10, idle: 2 }, + + // Handler-level defaults + 'batch-import': { min: 10, max: 100, idle: 20 }, + 'real-time': { min: 2, max: 10, idle: 3 }, + analytics: { min: 5, max: 30, idle: 10 }, + reporting: { min: 3, max: 20, idle: 5 }, + }; + + static calculate( + serviceName: string, + handlerName?: string, + customConfig?: Partial + ): PoolSizeRecommendation { + // Check for custom configuration first + if (customConfig?.minConnections && customConfig?.maxConnections) { + return { + min: customConfig.minConnections, + max: customConfig.maxConnections, + idle: Math.floor((customConfig.minConnections + customConfig.maxConnections) / 4), + }; + } + + // Try handler-specific sizes first, then service-level + const key = handlerName || serviceName; + const recommendation = this.DEFAULT_SIZES[key] || this.DEFAULT_SIZES[serviceName]; + + if (recommendation) { + return { ...recommendation }; + } + + // Fall back to generic defaults + return { + min: 2, + max: 10, + idle: 3, + }; + } + + static getOptimalPoolSize( + expectedConcurrency: number, + averageQueryTimeMs: number, + targetLatencyMs: number + ): number { + // Little's Law: L = λ * W + // L = number of connections needed + // λ = arrival rate (requests per second) + // W = average time in system (seconds) + + const requestsPerSecond = expectedConcurrency; + const averageTimeInSystem = averageQueryTimeMs / 1000; + + const minConnections = Math.ceil(requestsPerSecond * averageTimeInSystem); + + // Add buffer for burst traffic (20% overhead) + const recommendedSize = Math.ceil(minConnections * 1.2); + + // Ensure we meet target latency + const latencyBasedSize = Math.ceil( + expectedConcurrency * (averageQueryTimeMs / targetLatencyMs) + ); + + return Math.max(recommendedSize, latencyBasedSize, 2); // Minimum 2 connections + } +} diff --git a/libs/core/di/src/types.ts b/libs/core/di/src/types.ts index bca003b..2cf2b0f 100644 --- a/libs/core/di/src/types.ts +++ b/libs/core/di/src/types.ts @@ -1,68 +1,71 @@ -// Generic types to avoid circular dependencies -export interface GenericClientConfig { - [key: string]: any; -} - -export interface ConnectionPoolConfig { - name: string; - poolSize?: number; - minConnections?: number; - maxConnections?: number; - idleTimeoutMillis?: number; - connectionTimeoutMillis?: number; - enableMetrics?: boolean; -} - -export interface MongoDBPoolConfig extends ConnectionPoolConfig { - config: GenericClientConfig; -} - -export interface PostgreSQLPoolConfig extends ConnectionPoolConfig { - config: GenericClientConfig; -} - -export interface CachePoolConfig extends ConnectionPoolConfig { - config: GenericClientConfig; -} - -export interface QueuePoolConfig extends ConnectionPoolConfig { - config: GenericClientConfig; -} - -export interface ConnectionFactoryConfig { - service: string; - environment: 'development' | 'production' | 'test'; - pools?: { - mongodb?: Partial; - postgres?: Partial; - cache?: Partial; - queue?: Partial; - }; -} - -export interface ConnectionPool { - name: string; - client: T; - metrics: PoolMetrics; - health(): Promise; - dispose(): Promise; -} - -export interface PoolMetrics { - created: Date; - totalConnections: number; - activeConnections: number; - idleConnections: number; - waitingRequests: number; - errors: number; -} - -export interface ConnectionFactory { - createMongoDB(config: MongoDBPoolConfig): Promise>; - createPostgreSQL(config: PostgreSQLPoolConfig): Promise>; - createCache(config: CachePoolConfig): Promise>; - createQueue(config: QueuePoolConfig): Promise>; - getPool(type: 'mongodb' | 'postgres' | 'cache' | 'queue', name: string): ConnectionPool | undefined; - listPools(): Array<{ type: string; name: string; metrics: PoolMetrics }>; - disposeAll(): Promise; -} \ No newline at end of file +// Generic types to avoid circular dependencies +export interface GenericClientConfig { + [key: string]: any; +} + +export interface ConnectionPoolConfig { + name: string; + poolSize?: number; + minConnections?: number; + maxConnections?: number; + idleTimeoutMillis?: number; + connectionTimeoutMillis?: number; + enableMetrics?: boolean; +} + +export interface MongoDBPoolConfig extends ConnectionPoolConfig { + config: GenericClientConfig; +} + +export interface PostgreSQLPoolConfig extends ConnectionPoolConfig { + config: GenericClientConfig; +} + +export interface CachePoolConfig extends ConnectionPoolConfig { + config: GenericClientConfig; +} + +export interface QueuePoolConfig extends ConnectionPoolConfig { + config: GenericClientConfig; +} + +export interface ConnectionFactoryConfig { + service: string; + environment: 'development' | 'production' | 'test'; + pools?: { + mongodb?: Partial; + postgres?: Partial; + cache?: Partial; + queue?: Partial; + }; +} + +export interface ConnectionPool { + name: string; + client: T; + metrics: PoolMetrics; + health(): Promise; + dispose(): Promise; +} + +export interface PoolMetrics { + created: Date; + totalConnections: number; + activeConnections: number; + idleConnections: number; + waitingRequests: number; + errors: number; +} + +export interface ConnectionFactory { + createMongoDB(config: MongoDBPoolConfig): Promise>; + createPostgreSQL(config: PostgreSQLPoolConfig): Promise>; + createCache(config: CachePoolConfig): Promise>; + createQueue(config: QueuePoolConfig): Promise>; + getPool( + type: 'mongodb' | 'postgres' | 'cache' | 'queue', + name: string + ): ConnectionPool | undefined; + listPools(): Array<{ type: string; name: string; metrics: PoolMetrics }>; + disposeAll(): Promise; +} diff --git a/libs/core/di/test/di.test.ts b/libs/core/di/test/di.test.ts index 5843073..3e7bc1f 100644 --- a/libs/core/di/test/di.test.ts +++ b/libs/core/di/test/di.test.ts @@ -1,178 +1,183 @@ -/** - * Test DI library functionality - */ -import { test, expect, describe } from 'bun:test'; -import { ServiceContainer, ConnectionFactory, OperationContext, PoolSizeCalculator } from '../src/index'; - -describe('DI Library', () => { - test('ServiceContainer - sync resolution', () => { - const container = new ServiceContainer('test'); - - container.register({ - name: 'testService', - factory: () => ({ value: 'test' }), - singleton: true, - }); - - const service = container.resolve<{ value: string }>('testService'); - expect(service.value).toBe('test'); - }); - - test('ServiceContainer - async resolution', async () => { - const container = new ServiceContainer('test'); - - container.register({ - name: 'asyncService', - factory: async () => ({ value: 'async-test' }), - singleton: true, - }); - - const service = await container.resolveAsync<{ value: string }>('asyncService'); - expect(service.value).toBe('async-test'); - }); - - test('ServiceContainer - scoped container', () => { - const container = new ServiceContainer('test'); - - container.register({ - name: 'testService', - factory: () => ({ value: 'test' }), - singleton: true, - }); - - const scopedContainer = container.createScope(); - const service = scopedContainer.resolve<{ value: string }>('testService'); - expect(service.value).toBe('test'); - }); - - test('ServiceContainer - error on unregistered service', () => { - const container = new ServiceContainer('test'); - - expect(() => { - container.resolve('nonexistent'); - }).toThrow('Service nonexistent not registered'); - }); - - test('ServiceContainer - async service throws error on sync resolve', () => { - const container = new ServiceContainer('test'); - - container.register({ - name: 'asyncService', - factory: async () => ({ value: 'async' }), - singleton: true, - }); - - expect(() => { - container.resolve('asyncService'); - }).toThrow('Service asyncService is async. Use resolveAsync() instead.'); - }); - - test('ServiceContainer - disposal', async () => { - const container = new ServiceContainer('test'); - let disposed = false; - - container.register({ - name: 'disposableService', - factory: () => ({ value: 'test' }), - singleton: true, - dispose: async () => { - disposed = true; - }, - }); - - // Create instance - container.resolve('disposableService'); - - // Dispose container - await container.dispose(); - expect(disposed).toBe(true); - }); - - test('OperationContext - enhanced functionality', () => { - const container = new ServiceContainer('test'); - const context = OperationContext.create('test-handler', 'test-operation', { - container, - metadata: { userId: '123' }, - }); - - expect(context).toBeDefined(); - expect(context.logger).toBeDefined(); - expect(context.traceId).toBeDefined(); - expect(context.metadata.userId).toBe('123'); - expect(context.getExecutionTime()).toBeGreaterThanOrEqual(0); - }); - - test('OperationContext - service resolution', () => { - const container = new ServiceContainer('test'); - - container.register({ - name: 'testService', - factory: () => ({ value: 'resolved' }), - singleton: true, - }); - - const context = OperationContext.create('test-handler', 'test-operation', { - container, - }); - - const service = context.resolve<{ value: string }>('testService'); - expect(service.value).toBe('resolved'); - }); - - test('ConnectionFactory - creation', () => { - const factory = new ConnectionFactory({ - service: 'test', - environment: 'development', - }); - - expect(factory).toBeDefined(); - expect(factory.listPools()).toEqual([]); - }); - - test('OperationContext - creation', () => { - const container = new ServiceContainer('test'); - const context = OperationContext.create('test-handler', 'test-operation', { - container, - }); - - expect(context).toBeDefined(); - expect(context.logger).toBeDefined(); - }); - - test('OperationContext - child context', () => { - const context = OperationContext.create('test-handler', 'test-operation'); - const child = context.createChild('child-operation'); - - expect(child).toBeDefined(); - expect(child.logger).toBeDefined(); - }); - - test('PoolSizeCalculator - service defaults', () => { - const poolSize = PoolSizeCalculator.calculate('data-ingestion'); - expect(poolSize).toEqual({ min: 5, max: 50, idle: 10 }); - }); - - test('PoolSizeCalculator - handler defaults', () => { - const poolSize = PoolSizeCalculator.calculate('unknown-service', 'batch-import'); - expect(poolSize).toEqual({ min: 10, max: 100, idle: 20 }); - }); - - test('PoolSizeCalculator - fallback defaults', () => { - const poolSize = PoolSizeCalculator.calculate('unknown-service', 'unknown-handler'); - expect(poolSize).toEqual({ min: 2, max: 10, idle: 3 }); - }); - - test('PoolSizeCalculator - custom config', () => { - const poolSize = PoolSizeCalculator.calculate('test-service', undefined, { - minConnections: 5, - maxConnections: 15, - }); - expect(poolSize).toEqual({ min: 5, max: 15, idle: 5 }); - }); - - test('PoolSizeCalculator - optimal size calculation', () => { - const optimalSize = PoolSizeCalculator.getOptimalPoolSize(10, 100, 50); - expect(optimalSize).toBeGreaterThan(0); - expect(typeof optimalSize).toBe('number'); - }); -}); \ No newline at end of file +/** + * Test DI library functionality + */ +import { describe, expect, test } from 'bun:test'; +import { + ConnectionFactory, + OperationContext, + PoolSizeCalculator, + ServiceContainer, +} from '../src/index'; + +describe('DI Library', () => { + test('ServiceContainer - sync resolution', () => { + const container = new ServiceContainer('test'); + + container.register({ + name: 'testService', + factory: () => ({ value: 'test' }), + singleton: true, + }); + + const service = container.resolve<{ value: string }>('testService'); + expect(service.value).toBe('test'); + }); + + test('ServiceContainer - async resolution', async () => { + const container = new ServiceContainer('test'); + + container.register({ + name: 'asyncService', + factory: async () => ({ value: 'async-test' }), + singleton: true, + }); + + const service = await container.resolveAsync<{ value: string }>('asyncService'); + expect(service.value).toBe('async-test'); + }); + + test('ServiceContainer - scoped container', () => { + const container = new ServiceContainer('test'); + + container.register({ + name: 'testService', + factory: () => ({ value: 'test' }), + singleton: true, + }); + + const scopedContainer = container.createScope(); + const service = scopedContainer.resolve<{ value: string }>('testService'); + expect(service.value).toBe('test'); + }); + + test('ServiceContainer - error on unregistered service', () => { + const container = new ServiceContainer('test'); + + expect(() => { + container.resolve('nonexistent'); + }).toThrow('Service nonexistent not registered'); + }); + + test('ServiceContainer - async service throws error on sync resolve', () => { + const container = new ServiceContainer('test'); + + container.register({ + name: 'asyncService', + factory: async () => ({ value: 'async' }), + singleton: true, + }); + + expect(() => { + container.resolve('asyncService'); + }).toThrow('Service asyncService is async. Use resolveAsync() instead.'); + }); + + test('ServiceContainer - disposal', async () => { + const container = new ServiceContainer('test'); + let disposed = false; + + container.register({ + name: 'disposableService', + factory: () => ({ value: 'test' }), + singleton: true, + dispose: async () => { + disposed = true; + }, + }); + + // Create instance + container.resolve('disposableService'); + + // Dispose container + await container.dispose(); + expect(disposed).toBe(true); + }); + + test('OperationContext - enhanced functionality', () => { + const container = new ServiceContainer('test'); + const context = OperationContext.create('test-handler', 'test-operation', { + container, + metadata: { userId: '123' }, + }); + + expect(context).toBeDefined(); + expect(context.logger).toBeDefined(); + expect(context.traceId).toBeDefined(); + expect(context.metadata.userId).toBe('123'); + expect(context.getExecutionTime()).toBeGreaterThanOrEqual(0); + }); + + test('OperationContext - service resolution', () => { + const container = new ServiceContainer('test'); + + container.register({ + name: 'testService', + factory: () => ({ value: 'resolved' }), + singleton: true, + }); + + const context = OperationContext.create('test-handler', 'test-operation', { + container, + }); + + const service = context.resolve<{ value: string }>('testService'); + expect(service.value).toBe('resolved'); + }); + + test('ConnectionFactory - creation', () => { + const factory = new ConnectionFactory({ + service: 'test', + environment: 'development', + }); + + expect(factory).toBeDefined(); + expect(factory.listPools()).toEqual([]); + }); + + test('OperationContext - creation', () => { + const container = new ServiceContainer('test'); + const context = OperationContext.create('test-handler', 'test-operation', { + container, + }); + + expect(context).toBeDefined(); + expect(context.logger).toBeDefined(); + }); + + test('OperationContext - child context', () => { + const context = OperationContext.create('test-handler', 'test-operation'); + const child = context.createChild('child-operation'); + + expect(child).toBeDefined(); + expect(child.logger).toBeDefined(); + }); + + test('PoolSizeCalculator - service defaults', () => { + const poolSize = PoolSizeCalculator.calculate('data-ingestion'); + expect(poolSize).toEqual({ min: 5, max: 50, idle: 10 }); + }); + + test('PoolSizeCalculator - handler defaults', () => { + const poolSize = PoolSizeCalculator.calculate('unknown-service', 'batch-import'); + expect(poolSize).toEqual({ min: 10, max: 100, idle: 20 }); + }); + + test('PoolSizeCalculator - fallback defaults', () => { + const poolSize = PoolSizeCalculator.calculate('unknown-service', 'unknown-handler'); + expect(poolSize).toEqual({ min: 2, max: 10, idle: 3 }); + }); + + test('PoolSizeCalculator - custom config', () => { + const poolSize = PoolSizeCalculator.calculate('test-service', undefined, { + minConnections: 5, + maxConnections: 15, + }); + expect(poolSize).toEqual({ min: 5, max: 15, idle: 5 }); + }); + + test('PoolSizeCalculator - optimal size calculation', () => { + const optimalSize = PoolSizeCalculator.getOptimalPoolSize(10, 100, 50); + expect(optimalSize).toBeGreaterThan(0); + expect(typeof optimalSize).toBe('number'); + }); +}); diff --git a/libs/core/di/tsconfig.json b/libs/core/di/tsconfig.json index b1b5979..0177a72 100644 --- a/libs/core/di/tsconfig.json +++ b/libs/core/di/tsconfig.json @@ -1,17 +1,14 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "rootDir": "./src", - "outDir": "./dist", - "composite": true, - "declaration": true, - "declarationMap": true, - "types": ["node", "bun-types"] - }, - "include": ["src/**/*.ts"], - "exclude": ["node_modules", "dist", "test"], - "references": [ - { "path": "../config" }, - { "path": "../logger" } - ] -} \ No newline at end of file +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "rootDir": "./src", + "outDir": "./dist", + "composite": true, + "declaration": true, + "declarationMap": true, + "types": ["node", "bun-types"] + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "test"], + "references": [{ "path": "../config" }, { "path": "../logger" }] +} diff --git a/libs/core/handlers/package.json b/libs/core/handlers/package.json index 6f8551e..527cf0b 100644 --- a/libs/core/handlers/package.json +++ b/libs/core/handlers/package.json @@ -1,23 +1,23 @@ -{ - "name": "@stock-bot/handlers", - "version": "1.0.0", - "description": "Universal handler system for queue and event-driven operations", - "main": "./src/index.ts", - "types": "./src/index.ts", - "scripts": { - "build": "tsc", - "clean": "rimraf dist", - "test": "bun test" - }, - "dependencies": { - "@stock-bot/config": "workspace:*", - "@stock-bot/logger": "workspace:*", - "@stock-bot/types": "workspace:*", - "@stock-bot/di": "workspace:*" - }, - "devDependencies": { - "@types/node": "^20.11.0", - "typescript": "^5.3.0", - "bun-types": "^1.2.15" - } -} \ No newline at end of file +{ + "name": "@stock-bot/handlers", + "version": "1.0.0", + "description": "Universal handler system for queue and event-driven operations", + "main": "./src/index.ts", + "types": "./src/index.ts", + "scripts": { + "build": "tsc", + "clean": "rimraf dist", + "test": "bun test" + }, + "dependencies": { + "@stock-bot/config": "workspace:*", + "@stock-bot/logger": "workspace:*", + "@stock-bot/types": "workspace:*", + "@stock-bot/di": "workspace:*" + }, + "devDependencies": { + "@types/node": "^20.11.0", + "typescript": "^5.3.0", + "bun-types": "^1.2.15" + } +} diff --git a/libs/core/handlers/src/base/BaseHandler.ts b/libs/core/handlers/src/base/BaseHandler.ts index 05dfe6a..fe96461 100644 --- a/libs/core/handlers/src/base/BaseHandler.ts +++ b/libs/core/handlers/src/base/BaseHandler.ts @@ -1,297 +1,307 @@ -import { getLogger } from '@stock-bot/logger'; -import { createJobHandler, handlerRegistry, type HandlerConfigWithSchedule } from '@stock-bot/types'; -import { fetch } from '@stock-bot/utils'; -import type { Collection } from 'mongodb'; -import type { IServiceContainer } from '../types/service-container'; -import type { ExecutionContext, IHandler } from '../types/types'; - -/** - * Abstract base class for all handlers with improved DI - * Provides common functionality and structure for queue/event operations - */ -export abstract class BaseHandler implements IHandler { - // Direct service properties - flattened for cleaner access - readonly logger; - readonly cache; - readonly queue; - readonly proxy; - readonly browser; - readonly mongodb; - readonly postgres; - readonly questdb; - - private handlerName: string; - - constructor(services: IServiceContainer, handlerName?: string) { - // Flatten all services onto the handler instance - this.logger = getLogger(this.constructor.name); - this.cache = services.cache; - this.queue = services.queue; - this.proxy = services.proxy; - this.browser = services.browser; - this.mongodb = services.mongodb; - this.postgres = services.postgres; - this.questdb = services.questdb; - - // Read handler name from decorator first, then fallback to parameter or class name - const constructor = this.constructor as any; - this.handlerName = constructor.__handlerName || handlerName || this.constructor.name.toLowerCase(); - } - - /** - * Main execution method - automatically routes to decorated methods - * Works with queue (events commented for future) - */ - async execute(operation: string, input: unknown, context: ExecutionContext): Promise { - const constructor = this.constructor as any; - const operations = constructor.__operations || []; - - // Debug logging - this.logger.debug('Handler execute called', { - handler: this.handlerName, - operation, - availableOperations: operations.map((op: any) => ({ name: op.name, method: op.method })) - }); - - // Find the operation metadata - const operationMeta = operations.find((op: any) => op.name === operation); - if (!operationMeta) { - this.logger.error('Operation not found', { - requestedOperation: operation, - availableOperations: operations.map((op: any) => op.name) - }); - throw new Error(`Unknown operation: ${operation}`); - } - - // Get the method from the instance and call it - const method = (this as any)[operationMeta.method]; - if (typeof method !== 'function') { - throw new Error(`Operation method '${operationMeta.method}' not found on handler`); - } - - this.logger.debug('Executing operation method', { - operation, - method: operationMeta.method - }); - - return await method.call(this, input, context); - } - - async scheduleOperation(operation: string, payload: unknown, delay?: number): Promise { - if (!this.queue) { - throw new Error('Queue service is not available'); - } - const queue = this.queue.getQueue(this.handlerName); - const jobData = { - handler: this.handlerName, - operation, - payload - }; - await queue.add(operation, jobData, { delay }); - } - - - /** - * Create execution context for operations - */ - protected createExecutionContext(type: 'http' | 'queue' | 'scheduled', metadata: Record = {}): ExecutionContext { - return { - type, - metadata: { - ...metadata, - timestamp: Date.now(), - traceId: `${this.constructor.name}-${Date.now()}-${Math.random().toString(36).substr(2, 9)}` - } - }; - } - - /** - * Helper methods for common operations - */ - - /** - * Get a MongoDB collection with type safety - */ - protected collection(name: string): Collection { - if (!this.mongodb) { - throw new Error('MongoDB service is not available'); - } - return this.mongodb.collection(name); - } - - /** - * Set cache with handler-prefixed key - */ - protected async cacheSet(key: string, value: any, ttl?: number): Promise { - if (!this.cache) { - return; - } - return this.cache.set(`${this.handlerName}:${key}`, value, ttl); - } - - /** - * Get cache with handler-prefixed key - */ - protected async cacheGet(key: string): Promise { - if (!this.cache) { - return null; - } - return this.cache.get(`${this.handlerName}:${key}`); - } - - /** - * Delete cache with handler-prefixed key - */ - protected async cacheDel(key: string): Promise { - if (!this.cache) { - return; - } - return this.cache.del(`${this.handlerName}:${key}`); - } - - /** - * Schedule operation with delay in seconds - */ - protected async scheduleIn(operation: string, payload: unknown, delaySeconds: number): Promise { - return this.scheduleOperation(operation, payload, delaySeconds * 1000); - } - - /** - * Log with handler context - */ - protected log(level: 'info' | 'warn' | 'error' | 'debug', message: string, meta?: any): void { - this.logger[level](message, { handler: this.handlerName, ...meta }); - } - - /** - * HTTP client helper using fetch from utils - */ - protected get http() { - return { - get: (url: string, options?: any) => - fetch(url, { ...options, method: 'GET', logger: this.logger }), - post: (url: string, data?: any, options?: any) => - fetch(url, { - ...options, - method: 'POST', - body: JSON.stringify(data), - headers: { 'Content-Type': 'application/json', ...options?.headers }, - logger: this.logger - }), - put: (url: string, data?: any, options?: any) => - fetch(url, { - ...options, - method: 'PUT', - body: JSON.stringify(data), - headers: { 'Content-Type': 'application/json', ...options?.headers }, - logger: this.logger - }), - delete: (url: string, options?: any) => - fetch(url, { ...options, method: 'DELETE', logger: this.logger }), - }; - } - - /** - * Check if a service is available - */ - protected hasService(name: keyof IServiceContainer): boolean { - const service = this[name as keyof this]; - return service !== null; - } - - /** - * Event methods - commented for future - */ - // protected async publishEvent(eventName: string, payload: unknown): Promise { - // const eventBus = await this.container.resolveAsync('eventBus'); - // await eventBus.publish(eventName, payload); - // } - - /** - * Register this handler using decorator metadata - * Automatically reads @Handler, @Operation, and @QueueSchedule decorators - */ - register(): void { - const constructor = this.constructor as any; - const handlerName = constructor.__handlerName || this.handlerName; - const operations = constructor.__operations || []; - const schedules = constructor.__schedules || []; - - // Create operation handlers from decorator metadata - const operationHandlers: Record = {}; - for (const op of operations) { - operationHandlers[op.name] = createJobHandler(async (payload) => { - const context: ExecutionContext = { - type: 'queue', - metadata: { source: 'queue', timestamp: Date.now() } - }; - return await this.execute(op.name, payload, context); - }); - } - - // Create scheduled jobs from decorator metadata - const scheduledJobs = schedules.map((schedule: any) => { - // Find the operation name from the method name - const operation = operations.find((op: any) => op.method === schedule.operation); - return { - type: `${handlerName}-${schedule.operation}`, - operation: operation?.name || schedule.operation, - cronPattern: schedule.cronPattern, - priority: schedule.priority || 5, - immediately: schedule.immediately || false, - description: schedule.description || `${handlerName} ${schedule.operation}`, - payload: this.getScheduledJobPayload?.(schedule.operation), - }; - }); - - const config: HandlerConfigWithSchedule = { - name: handlerName, - operations: operationHandlers, - scheduledJobs, - }; - - handlerRegistry.registerWithSchedule(config); - this.logger.info('Handler registered using decorator metadata', { - handlerName, - operations: operations.map((op: any) => ({ name: op.name, method: op.method })), - scheduledJobs: scheduledJobs.map((job: any) => ({ - operation: job.operation, - cronPattern: job.cronPattern, - immediately: job.immediately - })) - }); - } - - /** - * Override this method to provide payloads for scheduled jobs - * @param operation The operation name that needs a payload - * @returns The payload for the scheduled job, or undefined - */ - protected getScheduledJobPayload?(operation: string): any; - - /** - * Lifecycle hooks - can be overridden by subclasses - */ - async onInit?(): Promise; - async onStart?(): Promise; - async onStop?(): Promise; - async onDispose?(): Promise; -} - - -/** - * Specialized handler for operations that have scheduled jobs - */ -export abstract class ScheduledHandler extends BaseHandler { - /** - * Get scheduled job configurations for this handler - * Override in subclasses to define schedules - */ - getScheduledJobs?(): Array<{ - operation: string; - cronPattern: string; - priority?: number; - immediately?: boolean; - description?: string; - }>; -} \ No newline at end of file +import type { Collection } from 'mongodb'; +import { getLogger } from '@stock-bot/logger'; +import { + createJobHandler, + handlerRegistry, + type HandlerConfigWithSchedule, +} from '@stock-bot/types'; +import { fetch } from '@stock-bot/utils'; +import type { IServiceContainer } from '../types/service-container'; +import type { ExecutionContext, IHandler } from '../types/types'; + +/** + * Abstract base class for all handlers with improved DI + * Provides common functionality and structure for queue/event operations + */ +export abstract class BaseHandler implements IHandler { + // Direct service properties - flattened for cleaner access + readonly logger; + readonly cache; + readonly queue; + readonly proxy; + readonly browser; + readonly mongodb; + readonly postgres; + readonly questdb; + + private handlerName: string; + + constructor(services: IServiceContainer, handlerName?: string) { + // Flatten all services onto the handler instance + this.logger = getLogger(this.constructor.name); + this.cache = services.cache; + this.queue = services.queue; + this.proxy = services.proxy; + this.browser = services.browser; + this.mongodb = services.mongodb; + this.postgres = services.postgres; + this.questdb = services.questdb; + + // Read handler name from decorator first, then fallback to parameter or class name + const constructor = this.constructor as any; + this.handlerName = + constructor.__handlerName || handlerName || this.constructor.name.toLowerCase(); + } + + /** + * Main execution method - automatically routes to decorated methods + * Works with queue (events commented for future) + */ + async execute(operation: string, input: unknown, context: ExecutionContext): Promise { + const constructor = this.constructor as any; + const operations = constructor.__operations || []; + + // Debug logging + this.logger.debug('Handler execute called', { + handler: this.handlerName, + operation, + availableOperations: operations.map((op: any) => ({ name: op.name, method: op.method })), + }); + + // Find the operation metadata + const operationMeta = operations.find((op: any) => op.name === operation); + if (!operationMeta) { + this.logger.error('Operation not found', { + requestedOperation: operation, + availableOperations: operations.map((op: any) => op.name), + }); + throw new Error(`Unknown operation: ${operation}`); + } + + // Get the method from the instance and call it + const method = (this as any)[operationMeta.method]; + if (typeof method !== 'function') { + throw new Error(`Operation method '${operationMeta.method}' not found on handler`); + } + + this.logger.debug('Executing operation method', { + operation, + method: operationMeta.method, + }); + + return await method.call(this, input, context); + } + + async scheduleOperation(operation: string, payload: unknown, delay?: number): Promise { + if (!this.queue) { + throw new Error('Queue service is not available'); + } + const queue = this.queue.getQueue(this.handlerName); + const jobData = { + handler: this.handlerName, + operation, + payload, + }; + await queue.add(operation, jobData, { delay }); + } + + /** + * Create execution context for operations + */ + protected createExecutionContext( + type: 'http' | 'queue' | 'scheduled', + metadata: Record = {} + ): ExecutionContext { + return { + type, + metadata: { + ...metadata, + timestamp: Date.now(), + traceId: `${this.constructor.name}-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`, + }, + }; + } + + /** + * Helper methods for common operations + */ + + /** + * Get a MongoDB collection with type safety + */ + protected collection(name: string): Collection { + if (!this.mongodb) { + throw new Error('MongoDB service is not available'); + } + return this.mongodb.collection(name); + } + + /** + * Set cache with handler-prefixed key + */ + protected async cacheSet(key: string, value: any, ttl?: number): Promise { + if (!this.cache) { + return; + } + return this.cache.set(`${this.handlerName}:${key}`, value, ttl); + } + + /** + * Get cache with handler-prefixed key + */ + protected async cacheGet(key: string): Promise { + if (!this.cache) { + return null; + } + return this.cache.get(`${this.handlerName}:${key}`); + } + + /** + * Delete cache with handler-prefixed key + */ + protected async cacheDel(key: string): Promise { + if (!this.cache) { + return; + } + return this.cache.del(`${this.handlerName}:${key}`); + } + + /** + * Schedule operation with delay in seconds + */ + protected async scheduleIn( + operation: string, + payload: unknown, + delaySeconds: number + ): Promise { + return this.scheduleOperation(operation, payload, delaySeconds * 1000); + } + + /** + * Log with handler context + */ + protected log(level: 'info' | 'warn' | 'error' | 'debug', message: string, meta?: any): void { + this.logger[level](message, { handler: this.handlerName, ...meta }); + } + + /** + * HTTP client helper using fetch from utils + */ + protected get http() { + return { + get: (url: string, options?: any) => + fetch(url, { ...options, method: 'GET', logger: this.logger }), + post: (url: string, data?: any, options?: any) => + fetch(url, { + ...options, + method: 'POST', + body: JSON.stringify(data), + headers: { 'Content-Type': 'application/json', ...options?.headers }, + logger: this.logger, + }), + put: (url: string, data?: any, options?: any) => + fetch(url, { + ...options, + method: 'PUT', + body: JSON.stringify(data), + headers: { 'Content-Type': 'application/json', ...options?.headers }, + logger: this.logger, + }), + delete: (url: string, options?: any) => + fetch(url, { ...options, method: 'DELETE', logger: this.logger }), + }; + } + + /** + * Check if a service is available + */ + protected hasService(name: keyof IServiceContainer): boolean { + const service = this[name as keyof this]; + return service !== null; + } + + /** + * Event methods - commented for future + */ + // protected async publishEvent(eventName: string, payload: unknown): Promise { + // const eventBus = await this.container.resolveAsync('eventBus'); + // await eventBus.publish(eventName, payload); + // } + + /** + * Register this handler using decorator metadata + * Automatically reads @Handler, @Operation, and @QueueSchedule decorators + */ + register(): void { + const constructor = this.constructor as any; + const handlerName = constructor.__handlerName || this.handlerName; + const operations = constructor.__operations || []; + const schedules = constructor.__schedules || []; + + // Create operation handlers from decorator metadata + const operationHandlers: Record = {}; + for (const op of operations) { + operationHandlers[op.name] = createJobHandler(async payload => { + const context: ExecutionContext = { + type: 'queue', + metadata: { source: 'queue', timestamp: Date.now() }, + }; + return await this.execute(op.name, payload, context); + }); + } + + // Create scheduled jobs from decorator metadata + const scheduledJobs = schedules.map((schedule: any) => { + // Find the operation name from the method name + const operation = operations.find((op: any) => op.method === schedule.operation); + return { + type: `${handlerName}-${schedule.operation}`, + operation: operation?.name || schedule.operation, + cronPattern: schedule.cronPattern, + priority: schedule.priority || 5, + immediately: schedule.immediately || false, + description: schedule.description || `${handlerName} ${schedule.operation}`, + payload: this.getScheduledJobPayload?.(schedule.operation), + }; + }); + + const config: HandlerConfigWithSchedule = { + name: handlerName, + operations: operationHandlers, + scheduledJobs, + }; + + handlerRegistry.registerWithSchedule(config); + this.logger.info('Handler registered using decorator metadata', { + handlerName, + operations: operations.map((op: any) => ({ name: op.name, method: op.method })), + scheduledJobs: scheduledJobs.map((job: any) => ({ + operation: job.operation, + cronPattern: job.cronPattern, + immediately: job.immediately, + })), + }); + } + + /** + * Override this method to provide payloads for scheduled jobs + * @param operation The operation name that needs a payload + * @returns The payload for the scheduled job, or undefined + */ + protected getScheduledJobPayload?(operation: string): any; + + /** + * Lifecycle hooks - can be overridden by subclasses + */ + async onInit?(): Promise; + async onStart?(): Promise; + async onStop?(): Promise; + async onDispose?(): Promise; +} + +/** + * Specialized handler for operations that have scheduled jobs + */ +export abstract class ScheduledHandler extends BaseHandler { + /** + * Get scheduled job configurations for this handler + * Override in subclasses to define schedules + */ + getScheduledJobs?(): Array<{ + operation: string; + cronPattern: string; + priority?: number; + immediately?: boolean; + description?: string; + }>; +} diff --git a/libs/core/handlers/src/decorators/decorators.ts b/libs/core/handlers/src/decorators/decorators.ts index 9a1dbda..102bfa2 100644 --- a/libs/core/handlers/src/decorators/decorators.ts +++ b/libs/core/handlers/src/decorators/decorators.ts @@ -1,148 +1,130 @@ -// Bun-compatible decorators (hybrid approach) - -/** - * Handler decorator - marks a class as a handler - * @param name Handler name for registration - */ -export function Handler(name: string) { - return function ( - target: T, - _context?: any - ) { - // Store handler name on the constructor - (target as any).__handlerName = name; - (target as any).__needsAutoRegistration = true; - - return target; - }; -} - -/** - * Operation decorator - marks a method as an operation - * @param name Operation name - */ -export function Operation(name: string): any { - return function ( - target: any, - methodName: string, - descriptor?: PropertyDescriptor - ): any { - // Store metadata directly on the class constructor - const constructor = target.constructor; - - if (!constructor.__operations) { - constructor.__operations = []; - } - constructor.__operations.push({ - name, - method: methodName, - }); - - return descriptor; - }; -} - -/** - * Queue schedule decorator - marks an operation as scheduled - * @param cronPattern Cron pattern for scheduling - * @param options Additional scheduling options - */ -export function QueueSchedule( - cronPattern: string, - options?: { - priority?: number; - immediately?: boolean; - description?: string; - } -): any { - return function ( - target: any, - methodName: string, - descriptor?: PropertyDescriptor - ): any { - // Store metadata directly on the class constructor - const constructor = target.constructor; - - if (!constructor.__schedules) { - constructor.__schedules = []; - } - constructor.__schedules.push({ - operation: methodName, - cronPattern, - ...options, - }); - - return descriptor; - }; -} - -/** - * Disabled decorator - marks a handler as disabled for auto-registration - * Handlers marked with @Disabled() will be skipped during auto-registration - */ -export function Disabled() { - return function ( - target: T, - _context?: any - ) { - // Store disabled flag on the constructor - (target as any).__disabled = true; - - return target; - }; -} - -/** - * Combined decorator for scheduled operations - * Automatically creates both an operation and a schedule - * @param name Operation name - * @param cronPattern Cron pattern for scheduling - * @param options Schedule options - */ -export function ScheduledOperation( - name: string, - cronPattern: string, - options?: { - priority?: number; - immediately?: boolean; - description?: string; - } -): any { - return function ( - target: any, - methodName: string, - descriptor?: PropertyDescriptor - ): any { - // Apply both decorators - Operation(name)(target, methodName, descriptor); - QueueSchedule(cronPattern, options)(target, methodName, descriptor); - return descriptor; - }; -} - -// Future event decorators - commented for now -// export function EventListener(eventName: string) { -// return function (target: any, propertyName: string, descriptor: PropertyDescriptor) { -// if (!target.constructor.__eventListeners) { -// target.constructor.__eventListeners = []; -// } -// target.constructor.__eventListeners.push({ -// eventName, -// method: propertyName, -// }); -// return descriptor; -// }; -// } - -// export function EventPublisher(eventName: string) { -// return function (target: any, propertyName: string, descriptor: PropertyDescriptor) { -// if (!target.constructor.__eventPublishers) { -// target.constructor.__eventPublishers = []; -// } -// target.constructor.__eventPublishers.push({ -// eventName, -// method: propertyName, -// }); -// return descriptor; -// }; -// } \ No newline at end of file +// Bun-compatible decorators (hybrid approach) + +/** + * Handler decorator - marks a class as a handler + * @param name Handler name for registration + */ +export function Handler(name: string) { + return function (target: T, _context?: any) { + // Store handler name on the constructor + (target as any).__handlerName = name; + (target as any).__needsAutoRegistration = true; + + return target; + }; +} + +/** + * Operation decorator - marks a method as an operation + * @param name Operation name + */ +export function Operation(name: string): any { + return function (target: any, methodName: string, descriptor?: PropertyDescriptor): any { + // Store metadata directly on the class constructor + const constructor = target.constructor; + + if (!constructor.__operations) { + constructor.__operations = []; + } + constructor.__operations.push({ + name, + method: methodName, + }); + + return descriptor; + }; +} + +/** + * Queue schedule decorator - marks an operation as scheduled + * @param cronPattern Cron pattern for scheduling + * @param options Additional scheduling options + */ +export function QueueSchedule( + cronPattern: string, + options?: { + priority?: number; + immediately?: boolean; + description?: string; + } +): any { + return function (target: any, methodName: string, descriptor?: PropertyDescriptor): any { + // Store metadata directly on the class constructor + const constructor = target.constructor; + + if (!constructor.__schedules) { + constructor.__schedules = []; + } + constructor.__schedules.push({ + operation: methodName, + cronPattern, + ...options, + }); + + return descriptor; + }; +} + +/** + * Disabled decorator - marks a handler as disabled for auto-registration + * Handlers marked with @Disabled() will be skipped during auto-registration + */ +export function Disabled() { + return function (target: T, _context?: any) { + // Store disabled flag on the constructor + (target as any).__disabled = true; + + return target; + }; +} + +/** + * Combined decorator for scheduled operations + * Automatically creates both an operation and a schedule + * @param name Operation name + * @param cronPattern Cron pattern for scheduling + * @param options Schedule options + */ +export function ScheduledOperation( + name: string, + cronPattern: string, + options?: { + priority?: number; + immediately?: boolean; + description?: string; + } +): any { + return function (target: any, methodName: string, descriptor?: PropertyDescriptor): any { + // Apply both decorators + Operation(name)(target, methodName, descriptor); + QueueSchedule(cronPattern, options)(target, methodName, descriptor); + return descriptor; + }; +} + +// Future event decorators - commented for now +// export function EventListener(eventName: string) { +// return function (target: any, propertyName: string, descriptor: PropertyDescriptor) { +// if (!target.constructor.__eventListeners) { +// target.constructor.__eventListeners = []; +// } +// target.constructor.__eventListeners.push({ +// eventName, +// method: propertyName, +// }); +// return descriptor; +// }; +// } + +// export function EventPublisher(eventName: string) { +// return function (target: any, propertyName: string, descriptor: PropertyDescriptor) { +// if (!target.constructor.__eventPublishers) { +// target.constructor.__eventPublishers = []; +// } +// target.constructor.__eventPublishers.push({ +// eventName, +// method: propertyName, +// }); +// return descriptor; +// }; +// } diff --git a/libs/core/handlers/src/index.ts b/libs/core/handlers/src/index.ts index 9659398..69868f2 100644 --- a/libs/core/handlers/src/index.ts +++ b/libs/core/handlers/src/index.ts @@ -1,31 +1,37 @@ -// Base handler classes -export { BaseHandler, ScheduledHandler } from './base/BaseHandler'; - -// Handler registry (re-exported from types to avoid circular deps) -export { handlerRegistry } from '@stock-bot/types'; - -// Types -export type { - ExecutionContext, - IHandler, - JobHandler, - ScheduledJob, - HandlerConfig, - HandlerConfigWithSchedule, - TypedJobHandler, - HandlerMetadata, - OperationMetadata, -} from './types/types'; - -export type { IServiceContainer } from './types/service-container'; - -export { createJobHandler } from './types/types'; - -// Decorators -export { Handler, Operation, QueueSchedule, ScheduledOperation, Disabled } from './decorators/decorators'; - -// Auto-registration utilities -export { autoRegisterHandlers, createAutoHandlerRegistry } from './registry/auto-register'; - -// Future exports - commented for now -// export { EventListener, EventPublisher } from './decorators/decorators'; \ No newline at end of file +// Base handler classes +export { BaseHandler, ScheduledHandler } from './base/BaseHandler'; + +// Handler registry (re-exported from types to avoid circular deps) +export { handlerRegistry } from '@stock-bot/types'; + +// Types +export type { + ExecutionContext, + IHandler, + JobHandler, + ScheduledJob, + HandlerConfig, + HandlerConfigWithSchedule, + TypedJobHandler, + HandlerMetadata, + OperationMetadata, +} from './types/types'; + +export type { IServiceContainer } from './types/service-container'; + +export { createJobHandler } from './types/types'; + +// Decorators +export { + Handler, + Operation, + QueueSchedule, + ScheduledOperation, + Disabled, +} from './decorators/decorators'; + +// Auto-registration utilities +export { autoRegisterHandlers, createAutoHandlerRegistry } from './registry/auto-register'; + +// Future exports - commented for now +// export { EventListener, EventPublisher } from './decorators/decorators'; diff --git a/libs/core/handlers/src/registry/HandlerRegistry.ts b/libs/core/handlers/src/registry/HandlerRegistry.ts index 4101bc1..e5b5c89 100644 --- a/libs/core/handlers/src/registry/HandlerRegistry.ts +++ b/libs/core/handlers/src/registry/HandlerRegistry.ts @@ -1,191 +1,193 @@ -import { getLogger } from '@stock-bot/logger'; -import type { JobHandler, HandlerConfig, HandlerConfigWithSchedule, ScheduledJob } from '../types/types'; - -const logger = getLogger('handler-registry'); - -class HandlerRegistry { - private handlers = new Map(); - private handlerSchedules = new Map(); - - /** - * Register a handler with its operations (simple config) - */ - register(handlerName: string, config: HandlerConfig): void { - logger.info(`Registering handler: ${handlerName}`, { - operations: Object.keys(config), - }); - - this.handlers.set(handlerName, config); - } - - /** - * Register a handler with operations and scheduled jobs (full config) - */ - registerWithSchedule(config: HandlerConfigWithSchedule): void { - logger.info(`Registering handler with schedule: ${config.name}`, { - operations: Object.keys(config.operations), - scheduledJobs: config.scheduledJobs?.length || 0, - }); - - this.handlers.set(config.name, config.operations); - - if (config.scheduledJobs && config.scheduledJobs.length > 0) { - this.handlerSchedules.set(config.name, config.scheduledJobs); - } - } - - /** - * Get a handler for a specific handler and operation - */ - getHandler(handler: string, operation: string): JobHandler | null { - const handlerConfig = this.handlers.get(handler); - if (!handlerConfig) { - logger.warn(`Handler not found: ${handler}`); - return null; - } - - const jobHandler = handlerConfig[operation]; - if (!jobHandler) { - logger.warn(`Operation not found: ${handler}:${operation}`, { - availableOperations: Object.keys(handlerConfig), - }); - return null; - } - - return jobHandler; - } - - /** - * Get all scheduled jobs from all handlers - */ - getAllScheduledJobs(): Array<{ handler: string; job: ScheduledJob }> { - const allJobs: Array<{ handler: string; job: ScheduledJob }> = []; - - for (const [handlerName, jobs] of this.handlerSchedules) { - for (const job of jobs) { - allJobs.push({ - handler: handlerName, - job, - }); - } - } - - return allJobs; - } - - /** - * Get scheduled jobs for a specific handler - */ - getScheduledJobs(handler: string): ScheduledJob[] { - return this.handlerSchedules.get(handler) || []; - } - - /** - * Check if a handler has scheduled jobs - */ - hasScheduledJobs(handler: string): boolean { - return this.handlerSchedules.has(handler); - } - - /** - * Get all registered handlers with their configurations - */ - getHandlerConfigs(): Array<{ name: string; operations: string[]; scheduledJobs: number }> { - return Array.from(this.handlers.keys()).map(name => ({ - name, - operations: Object.keys(this.handlers.get(name) || {}), - scheduledJobs: this.handlerSchedules.get(name)?.length || 0, - })); - } - - /** - * Get all handlers with their full configurations for queue manager registration - */ - getAllHandlers(): Map { - const result = new Map< - string, - { operations: HandlerConfig; scheduledJobs?: ScheduledJob[] } - >(); - - for (const [name, operations] of this.handlers) { - const scheduledJobs = this.handlerSchedules.get(name); - result.set(name, { - operations, - scheduledJobs, - }); - } - - return result; - } - - /** - * Get all registered handlers - */ - getHandlers(): string[] { - return Array.from(this.handlers.keys()); - } - - /** - * Get operations for a specific handler - */ - getOperations(handler: string): string[] { - const handlerConfig = this.handlers.get(handler); - return handlerConfig ? Object.keys(handlerConfig) : []; - } - - /** - * Check if a handler exists - */ - hasHandler(handler: string): boolean { - return this.handlers.has(handler); - } - - /** - * Check if a handler has a specific operation - */ - hasOperation(handler: string, operation: string): boolean { - const handlerConfig = this.handlers.get(handler); - return handlerConfig ? operation in handlerConfig : false; - } - - /** - * Remove a handler - */ - unregister(handler: string): boolean { - this.handlerSchedules.delete(handler); - return this.handlers.delete(handler); - } - - /** - * Clear all handlers - */ - clear(): void { - this.handlers.clear(); - this.handlerSchedules.clear(); - } - - /** - * Get registry statistics - */ - getStats(): { handlers: number; totalOperations: number; totalScheduledJobs: number } { - let totalOperations = 0; - let totalScheduledJobs = 0; - - for (const config of this.handlers.values()) { - totalOperations += Object.keys(config).length; - } - - for (const jobs of this.handlerSchedules.values()) { - totalScheduledJobs += jobs.length; - } - - return { - handlers: this.handlers.size, - totalOperations, - totalScheduledJobs, - }; - } -} - -// Export singleton instance -export const handlerRegistry = new HandlerRegistry(); \ No newline at end of file +import { getLogger } from '@stock-bot/logger'; +import type { + HandlerConfig, + HandlerConfigWithSchedule, + JobHandler, + ScheduledJob, +} from '../types/types'; + +const logger = getLogger('handler-registry'); + +class HandlerRegistry { + private handlers = new Map(); + private handlerSchedules = new Map(); + + /** + * Register a handler with its operations (simple config) + */ + register(handlerName: string, config: HandlerConfig): void { + logger.info(`Registering handler: ${handlerName}`, { + operations: Object.keys(config), + }); + + this.handlers.set(handlerName, config); + } + + /** + * Register a handler with operations and scheduled jobs (full config) + */ + registerWithSchedule(config: HandlerConfigWithSchedule): void { + logger.info(`Registering handler with schedule: ${config.name}`, { + operations: Object.keys(config.operations), + scheduledJobs: config.scheduledJobs?.length || 0, + }); + + this.handlers.set(config.name, config.operations); + + if (config.scheduledJobs && config.scheduledJobs.length > 0) { + this.handlerSchedules.set(config.name, config.scheduledJobs); + } + } + + /** + * Get a handler for a specific handler and operation + */ + getHandler(handler: string, operation: string): JobHandler | null { + const handlerConfig = this.handlers.get(handler); + if (!handlerConfig) { + logger.warn(`Handler not found: ${handler}`); + return null; + } + + const jobHandler = handlerConfig[operation]; + if (!jobHandler) { + logger.warn(`Operation not found: ${handler}:${operation}`, { + availableOperations: Object.keys(handlerConfig), + }); + return null; + } + + return jobHandler; + } + + /** + * Get all scheduled jobs from all handlers + */ + getAllScheduledJobs(): Array<{ handler: string; job: ScheduledJob }> { + const allJobs: Array<{ handler: string; job: ScheduledJob }> = []; + + for (const [handlerName, jobs] of this.handlerSchedules) { + for (const job of jobs) { + allJobs.push({ + handler: handlerName, + job, + }); + } + } + + return allJobs; + } + + /** + * Get scheduled jobs for a specific handler + */ + getScheduledJobs(handler: string): ScheduledJob[] { + return this.handlerSchedules.get(handler) || []; + } + + /** + * Check if a handler has scheduled jobs + */ + hasScheduledJobs(handler: string): boolean { + return this.handlerSchedules.has(handler); + } + + /** + * Get all registered handlers with their configurations + */ + getHandlerConfigs(): Array<{ name: string; operations: string[]; scheduledJobs: number }> { + return Array.from(this.handlers.keys()).map(name => ({ + name, + operations: Object.keys(this.handlers.get(name) || {}), + scheduledJobs: this.handlerSchedules.get(name)?.length || 0, + })); + } + + /** + * Get all handlers with their full configurations for queue manager registration + */ + getAllHandlers(): Map { + const result = new Map(); + + for (const [name, operations] of this.handlers) { + const scheduledJobs = this.handlerSchedules.get(name); + result.set(name, { + operations, + scheduledJobs, + }); + } + + return result; + } + + /** + * Get all registered handlers + */ + getHandlers(): string[] { + return Array.from(this.handlers.keys()); + } + + /** + * Get operations for a specific handler + */ + getOperations(handler: string): string[] { + const handlerConfig = this.handlers.get(handler); + return handlerConfig ? Object.keys(handlerConfig) : []; + } + + /** + * Check if a handler exists + */ + hasHandler(handler: string): boolean { + return this.handlers.has(handler); + } + + /** + * Check if a handler has a specific operation + */ + hasOperation(handler: string, operation: string): boolean { + const handlerConfig = this.handlers.get(handler); + return handlerConfig ? operation in handlerConfig : false; + } + + /** + * Remove a handler + */ + unregister(handler: string): boolean { + this.handlerSchedules.delete(handler); + return this.handlers.delete(handler); + } + + /** + * Clear all handlers + */ + clear(): void { + this.handlers.clear(); + this.handlerSchedules.clear(); + } + + /** + * Get registry statistics + */ + getStats(): { handlers: number; totalOperations: number; totalScheduledJobs: number } { + let totalOperations = 0; + let totalScheduledJobs = 0; + + for (const config of this.handlers.values()) { + totalOperations += Object.keys(config).length; + } + + for (const jobs of this.handlerSchedules.values()) { + totalScheduledJobs += jobs.length; + } + + return { + handlers: this.handlers.size, + totalOperations, + totalScheduledJobs, + }; + } +} + +// Export singleton instance +export const handlerRegistry = new HandlerRegistry(); diff --git a/libs/core/handlers/src/registry/auto-register.ts b/libs/core/handlers/src/registry/auto-register.ts index 274f647..a98e31a 100644 --- a/libs/core/handlers/src/registry/auto-register.ts +++ b/libs/core/handlers/src/registry/auto-register.ts @@ -1,180 +1,188 @@ -/** - * Auto-registration utilities for handlers - * Automatically discovers and registers handlers based on file patterns - */ - -import { getLogger } from '@stock-bot/logger'; -import type { IServiceContainer } from '../types/service-container'; -import { BaseHandler } from '../base/BaseHandler'; -import { readdirSync, statSync } from 'fs'; -import { join, relative } from 'path'; - -const logger = getLogger('handler-auto-register'); - -/** - * Recursively find all handler files in a directory - */ -function findHandlerFiles(dir: string, pattern = '.handler.'): string[] { - const files: string[] = []; - - function scan(currentDir: string) { - const entries = readdirSync(currentDir); - - for (const entry of entries) { - const fullPath = join(currentDir, entry); - const stat = statSync(fullPath); - - if (stat.isDirectory() && !entry.startsWith('.') && entry !== 'node_modules') { - scan(fullPath); - } else if (stat.isFile() && entry.includes(pattern) && entry.endsWith('.ts')) { - files.push(fullPath); - } - } - } - - scan(dir); - return files; -} - -/** - * Extract handler classes from a module - */ -function extractHandlerClasses(module: any): Array BaseHandler> { - const handlers: Array BaseHandler> = []; - - for (const key of Object.keys(module)) { - const exported = module[key]; - - // Check if it's a class that extends BaseHandler - if ( - typeof exported === 'function' && - exported.prototype && - exported.prototype instanceof BaseHandler - ) { - handlers.push(exported); - } - } - - return handlers; -} - -/** - * Auto-register all handlers in a directory - * @param directory The directory to scan for handlers - * @param services The service container to inject into handlers - * @param options Configuration options - */ -export async function autoRegisterHandlers( - directory: string, - services: IServiceContainer, - options: { - pattern?: string; - exclude?: string[]; - dryRun?: boolean; - } = {} -): Promise<{ registered: string[]; failed: string[] }> { - const { pattern = '.handler.', exclude = [], dryRun = false } = options; - const registered: string[] = []; - const failed: string[] = []; - - try { - logger.info('Starting auto-registration of handlers', { directory, pattern }); - - // Find all handler files - const handlerFiles = findHandlerFiles(directory, pattern); - logger.debug(`Found ${handlerFiles.length} handler files`, { files: handlerFiles }); - - // Process each handler file - for (const file of handlerFiles) { - const relativePath = relative(directory, file); - - // Skip excluded files - if (exclude.some(ex => relativePath.includes(ex))) { - logger.debug(`Skipping excluded file: ${relativePath}`); - continue; - } - - try { - // Import the module - const module = await import(file); - const handlerClasses = extractHandlerClasses(module); - - if (handlerClasses.length === 0) { - logger.warn(`No handler classes found in ${relativePath}`); - continue; - } - - // Register each handler class - for (const HandlerClass of handlerClasses) { - const handlerName = HandlerClass.name; - - // Check if handler is disabled - if ((HandlerClass as any).__disabled) { - logger.info(`Skipping disabled handler: ${handlerName} from ${relativePath}`); - continue; - } - - if (dryRun) { - logger.info(`[DRY RUN] Would register handler: ${handlerName} from ${relativePath}`); - registered.push(handlerName); - } else { - logger.info(`Registering handler: ${handlerName} from ${relativePath}`); - - // Create instance and register - const handler = new HandlerClass(services); - handler.register(); - - registered.push(handlerName); - logger.info(`Successfully registered handler: ${handlerName}`); - } - } - } catch (error) { - logger.error(`Failed to process handler file: ${relativePath}`, { error }); - failed.push(relativePath); - } - } - - logger.info('Auto-registration complete', { - totalFiles: handlerFiles.length, - registered: registered.length, - failed: failed.length - }); - - return { registered, failed }; - } catch (error) { - logger.error('Auto-registration failed', { error }); - throw error; - } -} - -/** - * Create a handler registry that auto-discovers handlers - */ -export function createAutoHandlerRegistry(services: IServiceContainer) { - return { - /** - * Register all handlers from a directory - */ - async registerDirectory(directory: string, options?: Parameters[2]) { - return autoRegisterHandlers(directory, services, options); - }, - - /** - * Register handlers from multiple directories - */ - async registerDirectories(directories: string[], options?: Parameters[2]) { - const results = { - registered: [] as string[], - failed: [] as string[] - }; - - for (const dir of directories) { - const result = await autoRegisterHandlers(dir, services, options); - results.registered.push(...result.registered); - results.failed.push(...result.failed); - } - - return results; - } - }; -} \ No newline at end of file +/** + * Auto-registration utilities for handlers + * Automatically discovers and registers handlers based on file patterns + */ + +import { readdirSync, statSync } from 'fs'; +import { join, relative } from 'path'; +import { getLogger } from '@stock-bot/logger'; +import { BaseHandler } from '../base/BaseHandler'; +import type { IServiceContainer } from '../types/service-container'; + +const logger = getLogger('handler-auto-register'); + +/** + * Recursively find all handler files in a directory + */ +function findHandlerFiles(dir: string, pattern = '.handler.'): string[] { + const files: string[] = []; + + function scan(currentDir: string) { + const entries = readdirSync(currentDir); + + for (const entry of entries) { + const fullPath = join(currentDir, entry); + const stat = statSync(fullPath); + + if (stat.isDirectory() && !entry.startsWith('.') && entry !== 'node_modules') { + scan(fullPath); + } else if (stat.isFile() && entry.includes(pattern) && entry.endsWith('.ts')) { + files.push(fullPath); + } + } + } + + scan(dir); + return files; +} + +/** + * Extract handler classes from a module + */ +function extractHandlerClasses( + module: any +): Array BaseHandler> { + const handlers: Array BaseHandler> = []; + + for (const key of Object.keys(module)) { + const exported = module[key]; + + // Check if it's a class that extends BaseHandler + if ( + typeof exported === 'function' && + exported.prototype && + exported.prototype instanceof BaseHandler + ) { + handlers.push(exported); + } + } + + return handlers; +} + +/** + * Auto-register all handlers in a directory + * @param directory The directory to scan for handlers + * @param services The service container to inject into handlers + * @param options Configuration options + */ +export async function autoRegisterHandlers( + directory: string, + services: IServiceContainer, + options: { + pattern?: string; + exclude?: string[]; + dryRun?: boolean; + } = {} +): Promise<{ registered: string[]; failed: string[] }> { + const { pattern = '.handler.', exclude = [], dryRun = false } = options; + const registered: string[] = []; + const failed: string[] = []; + + try { + logger.info('Starting auto-registration of handlers', { directory, pattern }); + + // Find all handler files + const handlerFiles = findHandlerFiles(directory, pattern); + logger.debug(`Found ${handlerFiles.length} handler files`, { files: handlerFiles }); + + // Process each handler file + for (const file of handlerFiles) { + const relativePath = relative(directory, file); + + // Skip excluded files + if (exclude.some(ex => relativePath.includes(ex))) { + logger.debug(`Skipping excluded file: ${relativePath}`); + continue; + } + + try { + // Import the module + const module = await import(file); + const handlerClasses = extractHandlerClasses(module); + + if (handlerClasses.length === 0) { + logger.warn(`No handler classes found in ${relativePath}`); + continue; + } + + // Register each handler class + for (const HandlerClass of handlerClasses) { + const handlerName = HandlerClass.name; + + // Check if handler is disabled + if ((HandlerClass as any).__disabled) { + logger.info(`Skipping disabled handler: ${handlerName} from ${relativePath}`); + continue; + } + + if (dryRun) { + logger.info(`[DRY RUN] Would register handler: ${handlerName} from ${relativePath}`); + registered.push(handlerName); + } else { + logger.info(`Registering handler: ${handlerName} from ${relativePath}`); + + // Create instance and register + const handler = new HandlerClass(services); + handler.register(); + + registered.push(handlerName); + logger.info(`Successfully registered handler: ${handlerName}`); + } + } + } catch (error) { + logger.error(`Failed to process handler file: ${relativePath}`, { error }); + failed.push(relativePath); + } + } + + logger.info('Auto-registration complete', { + totalFiles: handlerFiles.length, + registered: registered.length, + failed: failed.length, + }); + + return { registered, failed }; + } catch (error) { + logger.error('Auto-registration failed', { error }); + throw error; + } +} + +/** + * Create a handler registry that auto-discovers handlers + */ +export function createAutoHandlerRegistry(services: IServiceContainer) { + return { + /** + * Register all handlers from a directory + */ + async registerDirectory( + directory: string, + options?: Parameters[2] + ) { + return autoRegisterHandlers(directory, services, options); + }, + + /** + * Register handlers from multiple directories + */ + async registerDirectories( + directories: string[], + options?: Parameters[2] + ) { + const results = { + registered: [] as string[], + failed: [] as string[], + }; + + for (const dir of directories) { + const result = await autoRegisterHandlers(dir, services, options); + results.registered.push(...result.registered); + results.failed.push(...result.failed); + } + + return results; + }, + }; +} diff --git a/libs/core/handlers/src/types/service-container.ts b/libs/core/handlers/src/types/service-container.ts index a7fc1aa..00c1aed 100644 --- a/libs/core/handlers/src/types/service-container.ts +++ b/libs/core/handlers/src/types/service-container.ts @@ -1,27 +1,27 @@ -/** - * Universal Service Container for Handlers - * Simple, comprehensive container with all services available - */ - -import type { ProxyManager } from '@stock-bot/proxy'; - -/** - * Universal service container with all common services - * Designed to work across different service contexts (data-ingestion, processing, etc.) - */ -export interface IServiceContainer { - // Core infrastructure - readonly logger: any; // Logger instance - readonly cache?: any; // Cache provider (Redis/Dragonfly) - optional - readonly queue?: any; // Queue manager (BullMQ) - optional - readonly proxy?: ProxyManager; // Proxy manager service - optional (depends on cache) - readonly browser?: any; // Browser automation (Playwright) - - // Database clients - all optional to support selective enabling - readonly mongodb?: any; // MongoDB client - readonly postgres?: any; // PostgreSQL client - readonly questdb?: any; // QuestDB client (time-series) - - // Optional extensions for future use - readonly custom?: Record; -} \ No newline at end of file +/** + * Universal Service Container for Handlers + * Simple, comprehensive container with all services available + */ + +import type { ProxyManager } from '@stock-bot/proxy'; + +/** + * Universal service container with all common services + * Designed to work across different service contexts (data-ingestion, processing, etc.) + */ +export interface IServiceContainer { + // Core infrastructure + readonly logger: any; // Logger instance + readonly cache?: any; // Cache provider (Redis/Dragonfly) - optional + readonly queue?: any; // Queue manager (BullMQ) - optional + readonly proxy?: ProxyManager; // Proxy manager service - optional (depends on cache) + readonly browser?: any; // Browser automation (Playwright) + + // Database clients - all optional to support selective enabling + readonly mongodb?: any; // MongoDB client + readonly postgres?: any; // PostgreSQL client + readonly questdb?: any; // QuestDB client (time-series) + + // Optional extensions for future use + readonly custom?: Record; +} diff --git a/libs/core/handlers/src/types/types.ts b/libs/core/handlers/src/types/types.ts index a547ecd..d87e571 100644 --- a/libs/core/handlers/src/types/types.ts +++ b/libs/core/handlers/src/types/types.ts @@ -1,14 +1,14 @@ -// Re-export all handler types from the shared types package -export type { - ExecutionContext, - HandlerConfig, - HandlerConfigWithSchedule, - HandlerMetadata, - IHandler, - JobHandler, - OperationMetadata, - ScheduledJob, - TypedJobHandler, -} from '@stock-bot/types'; - -export { createJobHandler } from '@stock-bot/types'; \ No newline at end of file +// Re-export all handler types from the shared types package +export type { + ExecutionContext, + HandlerConfig, + HandlerConfigWithSchedule, + HandlerMetadata, + IHandler, + JobHandler, + OperationMetadata, + ScheduledJob, + TypedJobHandler, +} from '@stock-bot/types'; + +export { createJobHandler } from '@stock-bot/types'; diff --git a/libs/core/handlers/tsconfig.json b/libs/core/handlers/tsconfig.json index 565be8c..f7867c2 100644 --- a/libs/core/handlers/tsconfig.json +++ b/libs/core/handlers/tsconfig.json @@ -1,15 +1,15 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src", - "composite": true - }, - "include": ["src/**/*"], - "references": [ - { "path": "../config" }, - { "path": "../logger" }, - { "path": "../di" }, - { "path": "../../utils" } - ] -} \ No newline at end of file +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src", + "composite": true + }, + "include": ["src/**/*"], + "references": [ + { "path": "../config" }, + { "path": "../logger" }, + { "path": "../di" }, + { "path": "../../utils" } + ] +} diff --git a/libs/core/logger/src/logger.ts b/libs/core/logger/src/logger.ts index 97253c9..4ac1c4e 100644 --- a/libs/core/logger/src/logger.ts +++ b/libs/core/logger/src/logger.ts @@ -58,12 +58,12 @@ function createDestination( // Console: In-process pretty stream for dev (fast shutdown) if (config.logConsole && config.environment !== 'production') { const prettyStream = pretty({ - sync: true, // IMPORTANT: Make async to prevent blocking the event loop + sync: true, // IMPORTANT: Make async to prevent blocking the event loop colorize: true, translateTime: 'yyyy-mm-dd HH:MM:ss.l', messageFormat: '[{service}{childName}] {msg}', - singleLine: false, // This was causing logs to be on one line - hideObject: false, // Hide metadata objects + singleLine: false, // This was causing logs to be on one line + hideObject: false, // Hide metadata objects ignore: 'pid,hostname,service,environment,version,childName', errorLikeObjectKeys: ['err', 'error'], errorProps: 'message,stack,name,code', @@ -193,7 +193,6 @@ export class Logger { } } - // Simple log level methods trace(message: string | object, metadata?: LogMetadata): void { this.log('trace', message, metadata); diff --git a/libs/core/logger/tsconfig.json b/libs/core/logger/tsconfig.json index dbc9566..9405533 100644 --- a/libs/core/logger/tsconfig.json +++ b/libs/core/logger/tsconfig.json @@ -6,6 +6,5 @@ "composite": true }, "include": ["src/**/*"], - "references": [ - ] + "references": [] } diff --git a/libs/core/types/src/backtesting.ts b/libs/core/types/src/backtesting.ts index b7700c6..0aa505d 100644 --- a/libs/core/types/src/backtesting.ts +++ b/libs/core/types/src/backtesting.ts @@ -3,9 +3,9 @@ * Types for strategy backtesting and analysis */ -import type { TradeExecution, TradePerformance } from './trading'; import type { PortfolioAnalysis } from './portfolio'; -import type { RiskMetrics, DrawdownAnalysis } from './risk-metrics'; +import type { DrawdownAnalysis, RiskMetrics } from './risk-metrics'; +import type { TradeExecution, TradePerformance } from './trading'; /** * Backtesting results @@ -31,4 +31,4 @@ export interface BacktestResults { initialCapital: number; /** Final value */ finalValue: number; -} \ No newline at end of file +} diff --git a/libs/core/types/src/financial-statements.ts b/libs/core/types/src/financial-statements.ts index 20f5fdb..dd4bda6 100644 --- a/libs/core/types/src/financial-statements.ts +++ b/libs/core/types/src/financial-statements.ts @@ -13,7 +13,7 @@ export interface BalanceSheet { period: string; /** Currency */ currency: string; - + // Assets /** Total current assets */ totalCurrentAssets: number; @@ -29,7 +29,7 @@ export interface BalanceSheet { prepaidExpenses?: number; /** Other current assets */ otherCurrentAssets?: number; - + /** Total non-current assets */ totalNonCurrentAssets: number; /** Property, plant & equipment (net) */ @@ -42,10 +42,10 @@ export interface BalanceSheet { longTermInvestments?: number; /** Other non-current assets */ otherNonCurrentAssets?: number; - + /** Total assets */ totalAssets: number; - + // Liabilities /** Total current liabilities */ totalCurrentLiabilities: number; @@ -57,7 +57,7 @@ export interface BalanceSheet { accruedLiabilities?: number; /** Other current liabilities */ otherCurrentLiabilities?: number; - + /** Total non-current liabilities */ totalNonCurrentLiabilities: number; /** Long-term debt */ @@ -66,10 +66,10 @@ export interface BalanceSheet { deferredTaxLiabilities?: number; /** Other non-current liabilities */ otherNonCurrentLiabilities?: number; - + /** Total liabilities */ totalLiabilities: number; - + // Equity /** Total stockholders' equity */ totalStockholdersEquity: number; @@ -95,14 +95,14 @@ export interface IncomeStatement { period: string; /** Currency */ currency: string; - + /** Total revenue/net sales */ totalRevenue: number; /** Cost of goods sold */ costOfGoodsSold: number; /** Gross profit */ grossProfit: number; - + /** Operating expenses */ operatingExpenses: number; /** Research and development */ @@ -113,24 +113,24 @@ export interface IncomeStatement { depreciationAmortization?: number; /** Other operating expenses */ otherOperatingExpenses?: number; - + /** Operating income */ operatingIncome: number; - + /** Interest income */ interestIncome?: number; /** Interest expense */ interestExpense?: number; /** Other income/expense */ otherIncomeExpense?: number; - + /** Income before taxes */ incomeBeforeTaxes: number; /** Income tax expense */ incomeTaxExpense: number; /** Net income */ netIncome: number; - + /** Earnings per share (basic) */ earningsPerShareBasic: number; /** Earnings per share (diluted) */ @@ -151,7 +151,7 @@ export interface CashFlowStatement { period: string; /** Currency */ currency: string; - + // Operating Activities /** Net income */ netIncome: number; @@ -163,8 +163,8 @@ export interface CashFlowStatement { otherOperatingActivities?: number; /** Net cash from operating activities */ netCashFromOperatingActivities: number; - - // Investing Activities + + // Investing Activities /** Capital expenditures */ capitalExpenditures: number; /** Acquisitions */ @@ -175,7 +175,7 @@ export interface CashFlowStatement { otherInvestingActivities?: number; /** Net cash from investing activities */ netCashFromInvestingActivities: number; - + // Financing Activities /** Debt issuance/repayment */ debtIssuanceRepayment?: number; @@ -187,11 +187,11 @@ export interface CashFlowStatement { otherFinancingActivities?: number; /** Net cash from financing activities */ netCashFromFinancingActivities: number; - + /** Net change in cash */ netChangeInCash: number; /** Cash at beginning of period */ cashAtBeginningOfPeriod: number; /** Cash at end of period */ cashAtEndOfPeriod: number; -} \ No newline at end of file +} diff --git a/libs/core/types/src/handler-registry.ts b/libs/core/types/src/handler-registry.ts index cb230ed..fa7710d 100644 --- a/libs/core/types/src/handler-registry.ts +++ b/libs/core/types/src/handler-registry.ts @@ -1,111 +1,119 @@ -/** - * Handler Registry - Lightweight registry for queue handlers - * Moved here to avoid circular dependencies between handlers and queue - */ - -import type { JobHandler, HandlerConfig, HandlerConfigWithSchedule, ScheduledJob } from './handlers'; - -class HandlerRegistry { - private handlers = new Map(); - private handlerSchedules = new Map(); - - /** - * Register a handler with its operations (simple config) - */ - register(handlerName: string, config: HandlerConfig): void { - console.log(`Registering handler: ${handlerName}`, { - operations: Object.keys(config), - }); - - this.handlers.set(handlerName, config); - } - - /** - * Register a handler with scheduled jobs (enhanced config) - */ - registerWithSchedule(config: HandlerConfigWithSchedule): void { - console.log(`Registering handler with schedule: ${config.name}`, { - operations: Object.keys(config.operations), - scheduledJobs: config.scheduledJobs?.length || 0, - }); - - this.handlers.set(config.name, config.operations); - - if (config.scheduledJobs && config.scheduledJobs.length > 0) { - this.handlerSchedules.set(config.name, config.scheduledJobs); - } - } - - /** - * Get a specific handler's configuration - */ - getHandler(handlerName: string): HandlerConfig | undefined { - return this.handlers.get(handlerName); - } - - /** - * Get all registered handlers - */ - getAllHandlers(): Map { - return new Map(this.handlers); - } - - /** - * Get scheduled jobs for a handler - */ - getScheduledJobs(handlerName: string): ScheduledJob[] { - return this.handlerSchedules.get(handlerName) || []; - } - - /** - * Get all handlers with their scheduled jobs - */ - getAllHandlersWithSchedule(): Map { - const result = new Map(); - - for (const [name, operations] of this.handlers) { - result.set(name, { - operations, - scheduledJobs: this.handlerSchedules.get(name) || [] - }); - } - - return result; - } - - /** - * Get a specific operation from a handler - */ - getOperation(handlerName: string, operationName: string): JobHandler | undefined { - const handler = this.handlers.get(handlerName); - if (!handler) { - return undefined; - } - return handler[operationName]; - } - - /** - * Check if a handler is registered - */ - hasHandler(handlerName: string): boolean { - return this.handlers.has(handlerName); - } - - /** - * Get list of all registered handler names - */ - getHandlerNames(): string[] { - return Array.from(this.handlers.keys()); - } - - /** - * Clear all registrations (useful for testing) - */ - clear(): void { - this.handlers.clear(); - this.handlerSchedules.clear(); - } -} - -// Export singleton instance -export const handlerRegistry = new HandlerRegistry(); \ No newline at end of file +/** + * Handler Registry - Lightweight registry for queue handlers + * Moved here to avoid circular dependencies between handlers and queue + */ + +import type { + HandlerConfig, + HandlerConfigWithSchedule, + JobHandler, + ScheduledJob, +} from './handlers'; + +class HandlerRegistry { + private handlers = new Map(); + private handlerSchedules = new Map(); + + /** + * Register a handler with its operations (simple config) + */ + register(handlerName: string, config: HandlerConfig): void { + console.log(`Registering handler: ${handlerName}`, { + operations: Object.keys(config), + }); + + this.handlers.set(handlerName, config); + } + + /** + * Register a handler with scheduled jobs (enhanced config) + */ + registerWithSchedule(config: HandlerConfigWithSchedule): void { + console.log(`Registering handler with schedule: ${config.name}`, { + operations: Object.keys(config.operations), + scheduledJobs: config.scheduledJobs?.length || 0, + }); + + this.handlers.set(config.name, config.operations); + + if (config.scheduledJobs && config.scheduledJobs.length > 0) { + this.handlerSchedules.set(config.name, config.scheduledJobs); + } + } + + /** + * Get a specific handler's configuration + */ + getHandler(handlerName: string): HandlerConfig | undefined { + return this.handlers.get(handlerName); + } + + /** + * Get all registered handlers + */ + getAllHandlers(): Map { + return new Map(this.handlers); + } + + /** + * Get scheduled jobs for a handler + */ + getScheduledJobs(handlerName: string): ScheduledJob[] { + return this.handlerSchedules.get(handlerName) || []; + } + + /** + * Get all handlers with their scheduled jobs + */ + getAllHandlersWithSchedule(): Map< + string, + { operations: HandlerConfig; scheduledJobs: ScheduledJob[] } + > { + const result = new Map(); + + for (const [name, operations] of this.handlers) { + result.set(name, { + operations, + scheduledJobs: this.handlerSchedules.get(name) || [], + }); + } + + return result; + } + + /** + * Get a specific operation from a handler + */ + getOperation(handlerName: string, operationName: string): JobHandler | undefined { + const handler = this.handlers.get(handlerName); + if (!handler) { + return undefined; + } + return handler[operationName]; + } + + /** + * Check if a handler is registered + */ + hasHandler(handlerName: string): boolean { + return this.handlers.has(handlerName); + } + + /** + * Get list of all registered handler names + */ + getHandlerNames(): string[] { + return Array.from(this.handlers.keys()); + } + + /** + * Clear all registrations (useful for testing) + */ + clear(): void { + this.handlers.clear(); + this.handlerSchedules.clear(); + } +} + +// Export singleton instance +export const handlerRegistry = new HandlerRegistry(); diff --git a/libs/core/types/src/handlers.ts b/libs/core/types/src/handlers.ts index b0430ba..6022e72 100644 --- a/libs/core/types/src/handlers.ts +++ b/libs/core/types/src/handlers.ts @@ -1,83 +1,83 @@ -/** - * Handler and Queue Types - * Shared types for handler system and queue operations - */ - -// Generic execution context - decoupled from service implementations -export interface ExecutionContext { - type: 'http' | 'queue' | 'scheduled' | 'event'; - metadata: { - source?: string; - jobId?: string; - attempts?: number; - timestamp?: number; - traceId?: string; - [key: string]: unknown; - }; -} - -// Simple handler interface -export interface IHandler { - execute(operation: string, input: unknown, context: ExecutionContext): Promise; -} - -// Job handler type for queue operations -export interface JobHandler { - (payload: TPayload): Promise; -} - -// Type-safe wrapper for creating job handlers -export type TypedJobHandler = (payload: TPayload) => Promise; - -// Scheduled job configuration -export interface ScheduledJob { - type: string; - operation: string; - payload?: T; - cronPattern: string; - priority?: number; - description?: string; - immediately?: boolean; - delay?: number; -} - -// Handler configuration -export interface HandlerConfig { - [operation: string]: JobHandler; -} - -// Handler configuration with schedule -export interface HandlerConfigWithSchedule { - name: string; - operations: Record; - scheduledJobs?: ScheduledJob[]; -} - -// Handler metadata for registry -export interface HandlerMetadata { - name: string; - version?: string; - description?: string; - operations: string[]; - scheduledJobs?: ScheduledJob[]; -} - -// Operation metadata for decorators -export interface OperationMetadata { - name: string; - schedules?: string[]; - operation?: string; - description?: string; - validation?: (input: unknown) => boolean; -} - -/** - * Create a typed job handler with validation - */ -export function createJobHandler( - handler: TypedJobHandler -): JobHandler { - return async (payload: unknown): Promise => { - return handler(payload as TPayload); - }; -} \ No newline at end of file +/** + * Handler and Queue Types + * Shared types for handler system and queue operations + */ + +// Generic execution context - decoupled from service implementations +export interface ExecutionContext { + type: 'http' | 'queue' | 'scheduled' | 'event'; + metadata: { + source?: string; + jobId?: string; + attempts?: number; + timestamp?: number; + traceId?: string; + [key: string]: unknown; + }; +} + +// Simple handler interface +export interface IHandler { + execute(operation: string, input: unknown, context: ExecutionContext): Promise; +} + +// Job handler type for queue operations +export interface JobHandler { + (payload: TPayload): Promise; +} + +// Type-safe wrapper for creating job handlers +export type TypedJobHandler = (payload: TPayload) => Promise; + +// Scheduled job configuration +export interface ScheduledJob { + type: string; + operation: string; + payload?: T; + cronPattern: string; + priority?: number; + description?: string; + immediately?: boolean; + delay?: number; +} + +// Handler configuration +export interface HandlerConfig { + [operation: string]: JobHandler; +} + +// Handler configuration with schedule +export interface HandlerConfigWithSchedule { + name: string; + operations: Record; + scheduledJobs?: ScheduledJob[]; +} + +// Handler metadata for registry +export interface HandlerMetadata { + name: string; + version?: string; + description?: string; + operations: string[]; + scheduledJobs?: ScheduledJob[]; +} + +// Operation metadata for decorators +export interface OperationMetadata { + name: string; + schedules?: string[]; + operation?: string; + description?: string; + validation?: (input: unknown) => boolean; +} + +/** + * Create a typed job handler with validation + */ +export function createJobHandler( + handler: TypedJobHandler +): JobHandler { + return async (payload: unknown): Promise => { + return handler(payload as TPayload); + }; +} diff --git a/libs/core/types/src/helpers.ts b/libs/core/types/src/helpers.ts index f4cc2ed..835d73b 100644 --- a/libs/core/types/src/helpers.ts +++ b/libs/core/types/src/helpers.ts @@ -33,4 +33,4 @@ export interface HasVolume { */ export interface HasTimestamp { timestamp: number; -} \ No newline at end of file +} diff --git a/libs/core/types/src/market-data.ts b/libs/core/types/src/market-data.ts index ec82774..4d51fd7 100644 --- a/libs/core/types/src/market-data.ts +++ b/libs/core/types/src/market-data.ts @@ -104,4 +104,4 @@ export interface MarketRegime { trendDirection?: 'up' | 'down'; /** Volatility level */ volatilityLevel: 'low' | 'medium' | 'high'; -} \ No newline at end of file +} diff --git a/libs/core/types/src/options.ts b/libs/core/types/src/options.ts index ea7cd9b..b56fb5d 100644 --- a/libs/core/types/src/options.ts +++ b/libs/core/types/src/options.ts @@ -55,4 +55,4 @@ export interface GreeksCalculation { vega: number; /** Rho - interest rate sensitivity */ rho: number; -} \ No newline at end of file +} diff --git a/libs/core/types/src/portfolio.ts b/libs/core/types/src/portfolio.ts index d034e48..bbf137b 100644 --- a/libs/core/types/src/portfolio.ts +++ b/libs/core/types/src/portfolio.ts @@ -105,4 +105,4 @@ export interface KellyParams { averageLoss: number; /** Risk-free rate */ riskFreeRate?: number; -} \ No newline at end of file +} diff --git a/libs/core/types/src/risk-metrics.ts b/libs/core/types/src/risk-metrics.ts index b3d7ca8..7d8ac7b 100644 --- a/libs/core/types/src/risk-metrics.ts +++ b/libs/core/types/src/risk-metrics.ts @@ -83,4 +83,4 @@ export interface ReturnAnalysis { averagePositiveReturn: number; /** Average negative return */ averageNegativeReturn: number; -} \ No newline at end of file +} diff --git a/libs/core/types/src/technical-analysis.ts b/libs/core/types/src/technical-analysis.ts index cd3d2d7..cd86638 100644 --- a/libs/core/types/src/technical-analysis.ts +++ b/libs/core/types/src/technical-analysis.ts @@ -14,23 +14,23 @@ export interface TechnicalIndicators { /** Relative Strength Index */ rsi: number[]; /** MACD indicator */ - macd: { - macd: number[]; - signal: number[]; - histogram: number[]; + macd: { + macd: number[]; + signal: number[]; + histogram: number[]; }; /** Bollinger Bands */ - bollinger: { - upper: number[]; - middle: number[]; - lower: number[]; + bollinger: { + upper: number[]; + middle: number[]; + lower: number[]; }; /** Average True Range */ atr: number[]; /** Stochastic Oscillator */ - stochastic: { - k: number[]; - d: number[]; + stochastic: { + k: number[]; + d: number[]; }; /** Williams %R */ williams_r: number[]; @@ -106,4 +106,4 @@ export interface GARCHParameters { aic: number; /** BIC (Bayesian Information Criterion) */ bic: number; -} \ No newline at end of file +} diff --git a/libs/core/types/src/trading.ts b/libs/core/types/src/trading.ts index e7a3eac..4c8cf6f 100644 --- a/libs/core/types/src/trading.ts +++ b/libs/core/types/src/trading.ts @@ -59,4 +59,4 @@ export interface TradePerformance { grossLoss: number; /** Net profit */ netProfit: number; -} \ No newline at end of file +} diff --git a/libs/core/types/tsconfig.json b/libs/core/types/tsconfig.json index dbc9566..9405533 100644 --- a/libs/core/types/tsconfig.json +++ b/libs/core/types/tsconfig.json @@ -6,6 +6,5 @@ "composite": true }, "include": ["src/**/*"], - "references": [ - ] + "references": [] } diff --git a/libs/data/cache/src/index.ts b/libs/data/cache/src/index.ts index b95d2e7..56f476f 100644 --- a/libs/data/cache/src/index.ts +++ b/libs/data/cache/src/index.ts @@ -39,7 +39,13 @@ export function createCache(options: CacheOptions): CacheProvider { // Export types and classes export type { - CacheConfig, CacheKey, CacheOptions, CacheProvider, CacheStats, RedisConfig, SerializationOptions + CacheConfig, + CacheKey, + CacheOptions, + CacheProvider, + CacheStats, + RedisConfig, + SerializationOptions, } from './types'; export { RedisConnectionManager } from './connection-manager'; diff --git a/libs/data/cache/tsconfig.json b/libs/data/cache/tsconfig.json index 88fe818..55c59a8 100644 --- a/libs/data/cache/tsconfig.json +++ b/libs/data/cache/tsconfig.json @@ -6,7 +6,5 @@ "composite": true }, "include": ["src/**/*"], - "references": [ - { "path": "../../core/logger" } - ] + "references": [{ "path": "../../core/logger" }] } diff --git a/libs/data/mongodb/src/client.ts b/libs/data/mongodb/src/client.ts index dbd2863..38176ab 100644 --- a/libs/data/mongodb/src/client.ts +++ b/libs/data/mongodb/src/client.ts @@ -1,7 +1,13 @@ -import type { Logger } from '@stock-bot/core/logger'; import type { OptionalUnlessRequiredId } from 'mongodb'; import { Collection, Db, MongoClient } from 'mongodb'; -import type { ConnectionEvents, DocumentBase, DynamicPoolConfig, MongoDBClientConfig, PoolMetrics } from './types'; +import type { Logger } from '@stock-bot/core/logger'; +import type { + ConnectionEvents, + DocumentBase, + DynamicPoolConfig, + MongoDBClientConfig, + PoolMetrics, +} from './types'; /** * MongoDB Client for Stock Bot Data Service @@ -71,7 +77,7 @@ export class MongoDBClient { if (this.events?.onConnect) { await Promise.resolve(this.events.onConnect()); } - + // Fire pool created event if (this.events?.onPoolCreated) { await Promise.resolve(this.events.onPoolCreated()); @@ -89,12 +95,12 @@ export class MongoDBClient { } catch (error) { this.metrics.errors++; this.metrics.lastError = error instanceof Error ? error.message : 'Unknown error'; - + // Fire error event if (this.events?.onError) { await Promise.resolve(this.events.onError(error as Error)); } - + this.logger.error('MongoDB connection failed:', error); if (this.client) { await this.client.close(); @@ -123,12 +129,12 @@ export class MongoDBClient { this.isConnected = false; this.client = null; this.db = null; - + // Fire disconnect event if (this.events?.onDisconnect) { await Promise.resolve(this.events.onDisconnect()); } - + this.logger.info('Disconnected from MongoDB'); } catch (error) { this.logger.error('Error disconnecting from MongoDB:', error); @@ -206,13 +212,16 @@ export class MongoDBClient { let totalUpdated = 0; const errors: unknown[] = []; - this.logger.info(`Starting batch upsert operation [${collectionName}-${documents.length}][${operationId}]`, { - database: dbName, - collection: collectionName, - totalDocuments: documents.length, - uniqueKeys: keyFields, - chunkSize, - }); + this.logger.info( + `Starting batch upsert operation [${collectionName}-${documents.length}][${operationId}]`, + { + database: dbName, + collection: collectionName, + totalDocuments: documents.length, + uniqueKeys: keyFields, + chunkSize, + } + ); // Process documents in chunks to avoid memory issues for (let i = 0; i < documents.length; i += chunkSize) { @@ -422,7 +431,7 @@ export class MongoDBClient { getPoolMetrics(): PoolMetrics { // Update last used timestamp this.metrics.lastUsed = new Date(); - + // Note: MongoDB driver doesn't expose detailed pool metrics // These are estimates based on configuration return { ...this.metrics }; @@ -433,7 +442,7 @@ export class MongoDBClient { */ setDynamicPoolConfig(config: DynamicPoolConfig): void { this.dynamicPoolConfig = config; - + if (config.enabled && this.isConnected && !this.poolMonitorInterval) { this.startPoolMonitoring(); } else if (!config.enabled && this.poolMonitorInterval) { @@ -465,7 +474,7 @@ export class MongoDBClient { const { minSize, maxSize, scaleUpThreshold, scaleDownThreshold } = this.dynamicPoolConfig; const currentSize = this.metrics.totalConnections; - const utilization = ((this.metrics.activeConnections / currentSize) * 100); + const utilization = (this.metrics.activeConnections / currentSize) * 100; this.logger.debug('Pool utilization', { utilization: `${utilization.toFixed(1)}%`, @@ -477,13 +486,21 @@ export class MongoDBClient { if (utilization > scaleUpThreshold && currentSize < maxSize) { const newSize = Math.min(currentSize + this.dynamicPoolConfig.scaleUpIncrement, maxSize); await this.resizePool(newSize); - this.logger.info('Scaling up connection pool', { from: currentSize, to: newSize, utilization }); + this.logger.info('Scaling up connection pool', { + from: currentSize, + to: newSize, + utilization, + }); } // Scale down if utilization is low else if (utilization < scaleDownThreshold && currentSize > minSize) { const newSize = Math.max(currentSize - this.dynamicPoolConfig.scaleDownIncrement, minSize); await this.resizePool(newSize); - this.logger.info('Scaling down connection pool', { from: currentSize, to: newSize, utilization }); + this.logger.info('Scaling down connection pool', { + from: currentSize, + to: newSize, + utilization, + }); } } @@ -494,8 +511,10 @@ export class MongoDBClient { private async resizePool(newSize: number): Promise { // MongoDB doesn't support dynamic pool resizing // This is a placeholder for future implementation - this.logger.warn('Dynamic pool resizing not yet implemented for MongoDB', { requestedSize: newSize }); - + this.logger.warn('Dynamic pool resizing not yet implemented for MongoDB', { + requestedSize: newSize, + }); + // Update metrics to reflect desired state this.metrics.totalConnections = newSize; } @@ -514,7 +533,10 @@ export class MongoDBClient { // Create minimum connections by running parallel pings for (let i = 0; i < minSize; i++) { promises.push( - this.client.db(this.defaultDatabase).admin().ping() + this.client + .db(this.defaultDatabase) + .admin() + .ping() .then(() => { this.logger.debug(`Warmed up connection ${i + 1}/${minSize}`); }) diff --git a/libs/data/mongodb/src/index.ts b/libs/data/mongodb/src/index.ts index 7401222..5e93d61 100644 --- a/libs/data/mongodb/src/index.ts +++ b/libs/data/mongodb/src/index.ts @@ -9,14 +9,20 @@ export { MongoDBClient } from './client'; // Types export type { - AnalystReport, ConnectionEvents, DocumentBase, DynamicPoolConfig, EarningsTranscript, - ExchangeSourceMapping, - MasterExchange, - MongoDBClientConfig, - MongoDBConnectionOptions, - NewsArticle, PoolMetrics, RawDocument, - SecFiling, - SentimentData + AnalystReport, + ConnectionEvents, + DocumentBase, + DynamicPoolConfig, + EarningsTranscript, + ExchangeSourceMapping, + MasterExchange, + MongoDBClientConfig, + MongoDBConnectionOptions, + NewsArticle, + PoolMetrics, + RawDocument, + SecFiling, + SentimentData, } from './types'; // Note: Factory functions removed - use Awilix DI container instead diff --git a/libs/data/mongodb/tsconfig.json b/libs/data/mongodb/tsconfig.json index 79532f7..75d5929 100644 --- a/libs/data/mongodb/tsconfig.json +++ b/libs/data/mongodb/tsconfig.json @@ -6,8 +6,5 @@ "composite": true }, "include": ["src/**/*"], - "references": [ - { "path": "../../core/logger" }, - { "path": "../../core/types" } - ] + "references": [{ "path": "../../core/logger" }, { "path": "../../core/types" }] } diff --git a/libs/data/postgres/src/client.ts b/libs/data/postgres/src/client.ts index 1e43350..a2037c5 100644 --- a/libs/data/postgres/src/client.ts +++ b/libs/data/postgres/src/client.ts @@ -4,13 +4,13 @@ import { PostgreSQLHealthMonitor } from './health'; import { PostgreSQLQueryBuilder } from './query-builder'; import { PostgreSQLTransactionManager } from './transactions'; import type { + ConnectionEvents, + DynamicPoolConfig, + PoolMetrics, PostgreSQLClientConfig, PostgreSQLConnectionOptions, QueryResult, TransactionCallback, - PoolMetrics, - ConnectionEvents, - DynamicPoolConfig, } from './types'; /** @@ -32,7 +32,12 @@ export class PostgreSQLClient { private dynamicPoolConfig?: DynamicPoolConfig; private poolMonitorInterval?: NodeJS.Timeout; - constructor(config: PostgreSQLClientConfig, logger?: any, options?: PostgreSQLConnectionOptions, events?: ConnectionEvents) { + constructor( + config: PostgreSQLClientConfig, + logger?: any, + options?: PostgreSQLConnectionOptions, + events?: ConnectionEvents + ) { this.config = config; this.options = { retryAttempts: 3, @@ -45,7 +50,7 @@ export class PostgreSQLClient { this.logger = logger || console; this.healthMonitor = new PostgreSQLHealthMonitor(this); this.transactionManager = new PostgreSQLTransactionManager(this); - + this.metrics = { totalConnections: 0, activeConnections: 0, @@ -80,22 +85,22 @@ export class PostgreSQLClient { client.release(); this.isConnected = true; - + // Update metrics const poolConfig = this.config.poolSettings; this.metrics.totalConnections = poolConfig?.max || 10; this.metrics.idleConnections = poolConfig?.min || 2; - + // Fire connection event if (this.events?.onConnect) { await Promise.resolve(this.events.onConnect()); } - + // Fire pool created event if (this.events?.onPoolCreated) { await Promise.resolve(this.events.onPoolCreated()); } - + this.logger.info('Successfully connected to PostgreSQL', { poolSize: this.metrics.totalConnections, }); @@ -105,10 +110,10 @@ export class PostgreSQLClient { // Setup error handlers this.setupErrorHandlers(); - + // Setup pool event listeners for metrics this.setupPoolMetrics(); - + // Start dynamic pool monitoring if enabled if (this.dynamicPoolConfig?.enabled) { this.startPoolMonitoring(); @@ -119,12 +124,12 @@ export class PostgreSQLClient { lastError = error as Error; this.metrics.errors++; this.metrics.lastError = lastError.message; - + // Fire error event if (this.events?.onError) { await Promise.resolve(this.events.onError(lastError)); } - + this.logger.error(`PostgreSQL connection attempt ${attempt} failed:`, error); if (this.pool) { @@ -157,17 +162,17 @@ export class PostgreSQLClient { clearInterval(this.poolMonitorInterval); this.poolMonitorInterval = undefined; } - + this.healthMonitor.stop(); await this.pool.end(); this.isConnected = false; this.pool = null; - + // Fire disconnect event if (this.events?.onDisconnect) { await Promise.resolve(this.events.onDisconnect()); } - + this.logger.info('Disconnected from PostgreSQL'); } catch (error) { this.logger.error('Error disconnecting from PostgreSQL:', error); @@ -429,7 +434,6 @@ export class PostgreSQLClient { return this.pool; } - private buildPoolConfig(): any { return { host: this.config.host, @@ -481,7 +485,7 @@ export class PostgreSQLClient { getPoolMetrics(): PoolMetrics { // Update last used timestamp this.metrics.lastUsed = new Date(); - + // Update metrics from pool if available if (this.pool) { this.metrics.totalConnections = this.pool.totalCount; @@ -489,7 +493,7 @@ export class PostgreSQLClient { this.metrics.waitingRequests = this.pool.waitingCount; this.metrics.activeConnections = this.metrics.totalConnections - this.metrics.idleConnections; } - + return { ...this.metrics }; } @@ -498,7 +502,7 @@ export class PostgreSQLClient { */ setDynamicPoolConfig(config: DynamicPoolConfig): void { this.dynamicPoolConfig = config; - + if (config.enabled && this.isConnected && !this.poolMonitorInterval) { this.startPoolMonitoring(); } else if (!config.enabled && this.poolMonitorInterval) { @@ -552,7 +556,7 @@ export class PostgreSQLClient { const metrics = this.getPoolMetrics(); const { minSize, maxSize, scaleUpThreshold, scaleDownThreshold } = this.dynamicPoolConfig; const currentSize = metrics.totalConnections; - const utilization = currentSize > 0 ? ((metrics.activeConnections / currentSize) * 100) : 0; + const utilization = currentSize > 0 ? (metrics.activeConnections / currentSize) * 100 : 0; this.logger.debug('Pool utilization', { utilization: `${utilization.toFixed(1)}%`, @@ -564,13 +568,21 @@ export class PostgreSQLClient { // Scale up if utilization is high or there are waiting requests if ((utilization > scaleUpThreshold || metrics.waitingRequests > 0) && currentSize < maxSize) { const newSize = Math.min(currentSize + this.dynamicPoolConfig.scaleUpIncrement, maxSize); - this.logger.info('Would scale up connection pool', { from: currentSize, to: newSize, utilization }); + this.logger.info('Would scale up connection pool', { + from: currentSize, + to: newSize, + utilization, + }); // Note: pg module doesn't support dynamic resizing, would need reconnection } // Scale down if utilization is low else if (utilization < scaleDownThreshold && currentSize > minSize) { const newSize = Math.max(currentSize - this.dynamicPoolConfig.scaleDownIncrement, minSize); - this.logger.info('Would scale down connection pool', { from: currentSize, to: newSize, utilization }); + this.logger.info('Would scale down connection pool', { + from: currentSize, + to: newSize, + utilization, + }); // Note: pg module doesn't support dynamic resizing, would need reconnection } } @@ -589,7 +601,8 @@ export class PostgreSQLClient { // Create minimum connections by running parallel queries for (let i = 0; i < minSize; i++) { promises.push( - this.pool.query('SELECT 1') + this.pool + .query('SELECT 1') .then(() => { this.logger.debug(`Warmed up connection ${i + 1}/${minSize}`); }) diff --git a/libs/data/postgres/tsconfig.json b/libs/data/postgres/tsconfig.json index 79532f7..75d5929 100644 --- a/libs/data/postgres/tsconfig.json +++ b/libs/data/postgres/tsconfig.json @@ -6,8 +6,5 @@ "composite": true }, "include": ["src/**/*"], - "references": [ - { "path": "../../core/logger" }, - { "path": "../../core/types" } - ] + "references": [{ "path": "../../core/logger" }, { "path": "../../core/types" }] } diff --git a/libs/data/questdb/src/client.ts b/libs/data/questdb/src/client.ts index ef9411c..8fac7a9 100644 --- a/libs/data/questdb/src/client.ts +++ b/libs/data/questdb/src/client.ts @@ -41,12 +41,12 @@ export class QuestDBClient { console.log('DEBUG: QuestDB client constructor called with config:', { ...config, user: config.user || '[NOT PROVIDED]', - password: config.password ? '[PROVIDED]' : '[NOT PROVIDED]' + password: config.password ? '[PROVIDED]' : '[NOT PROVIDED]', }); this.logger.debug('QuestDB client created with config:', { ...config, user: config.user || '[NOT PROVIDED]', - password: config.password ? '[PROVIDED]' : '[NOT PROVIDED]' + password: config.password ? '[PROVIDED]' : '[NOT PROVIDED]', }); this.healthMonitor = new QuestDBHealthMonitor(this); @@ -417,7 +417,6 @@ export class QuestDBClient { return { ...this.config }; } - private buildPgPoolConfig(): any { const config: any = { host: this.config.host, @@ -443,7 +442,7 @@ export class QuestDBClient { console.log('DEBUG: No user provided for QuestDB connection'); this.logger.debug('No user provided for QuestDB connection'); } - + if (this.config.password) { console.log('DEBUG: Adding password to QuestDB pool config'); this.logger.debug('Adding password to QuestDB pool config'); @@ -453,8 +452,14 @@ export class QuestDBClient { this.logger.debug('No password provided for QuestDB connection'); } - console.log('DEBUG: Final QuestDB pool config:', { ...config, password: config.password ? '[REDACTED]' : undefined }); - this.logger.debug('Final QuestDB pool config:', { ...config, password: config.password ? '[REDACTED]' : undefined }); + console.log('DEBUG: Final QuestDB pool config:', { + ...config, + password: config.password ? '[REDACTED]' : undefined, + }); + this.logger.debug('Final QuestDB pool config:', { + ...config, + password: config.password ? '[REDACTED]' : undefined, + }); return config; } diff --git a/libs/data/questdb/src/query-builder.ts b/libs/data/questdb/src/query-builder.ts index 2dc1a19..7770ac0 100644 --- a/libs/data/questdb/src/query-builder.ts +++ b/libs/data/questdb/src/query-builder.ts @@ -1,9 +1,5 @@ import { getLogger } from '@stock-bot/logger'; -import type { - QueryResult, - TableNames, - TimeRange, -} from './types'; +import type { QueryResult, TableNames, TimeRange } from './types'; // Interface to avoid circular dependency interface QuestDBClientInterface { diff --git a/libs/data/questdb/src/schema.ts b/libs/data/questdb/src/schema.ts index e584e5e..be91b3e 100644 --- a/libs/data/questdb/src/schema.ts +++ b/libs/data/questdb/src/schema.ts @@ -337,7 +337,6 @@ export class QuestDBSchemaManager { return sql; } - /** * Validate schema definition */ diff --git a/libs/data/questdb/tsconfig.json b/libs/data/questdb/tsconfig.json index 79532f7..75d5929 100644 --- a/libs/data/questdb/tsconfig.json +++ b/libs/data/questdb/tsconfig.json @@ -6,8 +6,5 @@ "composite": true }, "include": ["src/**/*"], - "references": [ - { "path": "../../core/logger" }, - { "path": "../../core/types" } - ] + "references": [{ "path": "../../core/logger" }, { "path": "../../core/types" }] } diff --git a/libs/services/browser/src/browser.ts b/libs/services/browser/src/browser.ts index f413ec7..7295048 100644 --- a/libs/services/browser/src/browser.ts +++ b/libs/services/browser/src/browser.ts @@ -174,9 +174,11 @@ export class Browser { if (proxy) { const [protocol, rest] = proxy.split('://'); if (!rest) { - throw new Error('Invalid proxy format. Expected protocol://host:port or protocol://user:pass@host:port'); + throw new Error( + 'Invalid proxy format. Expected protocol://host:port or protocol://user:pass@host:port' + ); } - + const [auth, hostPort] = rest.includes('@') ? rest.split('@') : [null, rest]; const finalHostPort = hostPort || rest; const [host, port] = finalHostPort.split(':'); diff --git a/libs/services/browser/tsconfig.json b/libs/services/browser/tsconfig.json index 88fe818..55c59a8 100644 --- a/libs/services/browser/tsconfig.json +++ b/libs/services/browser/tsconfig.json @@ -6,7 +6,5 @@ "composite": true }, "include": ["src/**/*"], - "references": [ - { "path": "../../core/logger" } - ] + "references": [{ "path": "../../core/logger" }] } diff --git a/libs/services/event-bus/src/event-bus.ts b/libs/services/event-bus/src/event-bus.ts index 749f613..2a40b3a 100644 --- a/libs/services/event-bus/src/event-bus.ts +++ b/libs/services/event-bus/src/event-bus.ts @@ -1,12 +1,7 @@ import { EventEmitter } from 'eventemitter3'; import Redis from 'ioredis'; import { getLogger } from '@stock-bot/logger'; -import type { - EventBusConfig, - EventBusMessage, - EventHandler, - EventSubscription, -} from './types'; +import type { EventBusConfig, EventBusMessage, EventHandler, EventSubscription } from './types'; /** * Lightweight Event Bus for inter-service communication @@ -52,7 +47,7 @@ export class EventBus extends EventEmitter { this.isConnected = true; }); - this.publisher.on('error', (error) => { + this.publisher.on('error', error => { this.logger.error('Publisher Redis error:', error); }); @@ -63,7 +58,7 @@ export class EventBus extends EventEmitter { this.resubscribeAll(); }); - this.subscriber.on('error', (error) => { + this.subscriber.on('error', error => { this.logger.error('Subscriber Redis error:', error); }); @@ -89,7 +84,7 @@ export class EventBus extends EventEmitter { // Call registered handler if exists const subscription = this.subscriptions.get(eventType); if (subscription?.handler) { - Promise.resolve(subscription.handler(eventMessage)).catch((error) => { + Promise.resolve(subscription.handler(eventMessage)).catch(error => { this.logger.error(`Handler error for event ${eventType}:`, error); }); } @@ -103,11 +98,7 @@ export class EventBus extends EventEmitter { /** * Publish an event */ - async publish( - type: string, - data: T, - metadata?: Record - ): Promise { + async publish(type: string, data: T, metadata?: Record): Promise { const message: EventBusMessage = { id: this.generateId(), type, @@ -199,11 +190,11 @@ export class EventBus extends EventEmitter { */ async waitForConnection(timeout: number = 5000): Promise { const startTime = Date.now(); - + while (!this.isConnected && Date.now() - startTime < timeout) { await new Promise(resolve => setTimeout(resolve, 100)); } - + if (!this.isConnected) { throw new Error(`Failed to connect to Redis within ${timeout}ms`); } @@ -220,10 +211,7 @@ export class EventBus extends EventEmitter { this.removeAllListeners(); // Close Redis connections - await Promise.all([ - this.publisher.quit(), - this.subscriber.quit(), - ]); + await Promise.all([this.publisher.quit(), this.subscriber.quit()]); this.logger.info('Event bus closed'); } @@ -248,4 +236,4 @@ export class EventBus extends EventEmitter { get service(): string { return this.serviceName; } -} \ No newline at end of file +} diff --git a/libs/services/event-bus/src/index.ts b/libs/services/event-bus/src/index.ts index 2bc77db..87dfbd2 100644 --- a/libs/services/event-bus/src/index.ts +++ b/libs/services/event-bus/src/index.ts @@ -10,4 +10,4 @@ export function createEventBus(config: EventBusConfig): EventBus { // Re-export everything export { EventBus } from './event-bus'; -export * from './types'; \ No newline at end of file +export * from './types'; diff --git a/libs/services/event-bus/src/types.ts b/libs/services/event-bus/src/types.ts index d07d569..07b8f53 100644 --- a/libs/services/event-bus/src/types.ts +++ b/libs/services/event-bus/src/types.ts @@ -33,27 +33,27 @@ export enum TradingEventType { PRICE_UPDATE = 'market.price.update', ORDERBOOK_UPDATE = 'market.orderbook.update', TRADE_EXECUTED = 'market.trade.executed', - + // Order events ORDER_CREATED = 'order.created', ORDER_FILLED = 'order.filled', ORDER_CANCELLED = 'order.cancelled', ORDER_REJECTED = 'order.rejected', - + // Position events POSITION_OPENED = 'position.opened', POSITION_CLOSED = 'position.closed', POSITION_UPDATED = 'position.updated', - + // Strategy events STRATEGY_SIGNAL = 'strategy.signal', STRATEGY_STARTED = 'strategy.started', STRATEGY_STOPPED = 'strategy.stopped', - + // Risk events RISK_LIMIT_BREACH = 'risk.limit.breach', RISK_WARNING = 'risk.warning', - + // System events SERVICE_STARTED = 'system.service.started', SERVICE_STOPPED = 'system.service.stopped', @@ -108,4 +108,4 @@ export interface RiskEvent { portfolioId?: string; strategyId?: string; message: string; -} \ No newline at end of file +} diff --git a/libs/services/event-bus/tsconfig.json b/libs/services/event-bus/tsconfig.json index 88fe818..55c59a8 100644 --- a/libs/services/event-bus/tsconfig.json +++ b/libs/services/event-bus/tsconfig.json @@ -6,7 +6,5 @@ "composite": true }, "include": ["src/**/*"], - "references": [ - { "path": "../../core/logger" } - ] + "references": [{ "path": "../../core/logger" }] } diff --git a/libs/services/proxy/package.json b/libs/services/proxy/package.json index 677fcde..6bb7edc 100644 --- a/libs/services/proxy/package.json +++ b/libs/services/proxy/package.json @@ -1,25 +1,25 @@ -{ - "name": "@stock-bot/proxy", - "version": "0.1.0", - "description": "Proxy management and synchronization services", - "main": "dist/index.js", - "types": "dist/index.d.ts", - "scripts": { - "build": "tsc", - "dev": "tsc --watch", - "clean": "rm -rf dist" - }, - "dependencies": { - "@stock-bot/logger": "workspace:*", - "@stock-bot/cache": "workspace:*" - }, - "devDependencies": { - "typescript": "^5.0.0" - }, - "exports": { - ".": { - "types": "./dist/index.d.ts", - "default": "./dist/index.js" - } - } -} \ No newline at end of file +{ + "name": "@stock-bot/proxy", + "version": "0.1.0", + "description": "Proxy management and synchronization services", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "tsc", + "dev": "tsc --watch", + "clean": "rm -rf dist" + }, + "dependencies": { + "@stock-bot/logger": "workspace:*", + "@stock-bot/cache": "workspace:*" + }, + "devDependencies": { + "typescript": "^5.0.0" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + } +} diff --git a/libs/services/proxy/src/index.ts b/libs/services/proxy/src/index.ts index 17ad05f..e224cfc 100644 --- a/libs/services/proxy/src/index.ts +++ b/libs/services/proxy/src/index.ts @@ -1,19 +1,16 @@ -/** - * Proxy Service Library - * Centralized proxy management and synchronization - */ - -// Main classes -export { ProxyManager } from './proxy-manager'; - -// Types -export type { - ProxyInfo, - ProxyManagerConfig, ProxyStats, ProxySyncConfig -} from './types'; - -// Note: Convenience functions removed as ProxyManager is no longer a singleton -// Create an instance and use its methods directly - -// Default export -export { ProxyManager as default } from './proxy-manager'; +/** + * Proxy Service Library + * Centralized proxy management and synchronization + */ + +// Main classes +export { ProxyManager } from './proxy-manager'; + +// Types +export type { ProxyInfo, ProxyManagerConfig, ProxyStats, ProxySyncConfig } from './types'; + +// Note: Convenience functions removed as ProxyManager is no longer a singleton +// Create an instance and use its methods directly + +// Default export +export { ProxyManager as default } from './proxy-manager'; diff --git a/libs/services/proxy/src/proxy-manager.ts b/libs/services/proxy/src/proxy-manager.ts index fc5fd74..f1019a2 100644 --- a/libs/services/proxy/src/proxy-manager.ts +++ b/libs/services/proxy/src/proxy-manager.ts @@ -1,284 +1,287 @@ -/** - * Centralized Proxy Manager - Handles proxy storage, retrieval, and caching - */ -import type { CacheProvider } from '@stock-bot/cache'; -import type { ProxyInfo, ProxyManagerConfig, ProxyStats } from './types'; - -export class ProxyManager { - private cache: CacheProvider; - private proxies: ProxyInfo[] = []; - private proxyIndex: number = 0; - private lastUpdate: Date | null = null; - private isInitialized = false; - private logger: any; - - constructor(cache: CacheProvider, _config: ProxyManagerConfig = {}, logger?: any) { - this.cache = cache; - this.logger = logger || console; - // Config can be used in the future for customization - } - - /** - * Internal initialization - loads existing proxies from cache - */ - private async initializeInternal(): Promise { - if (this.isInitialized) { - return; - } - - try { - this.logger.info('Initializing proxy manager...'); - - // Wait for cache to be ready - await this.cache.waitForReady(10000); // Wait up to 10 seconds - this.logger.debug('Cache is ready'); - - await this.loadFromCache(); - this.isInitialized = true; - this.logger.info('Proxy manager initialized', { - proxiesLoaded: this.proxies.length, - lastUpdate: this.lastUpdate, - }); - } catch (error) { - this.logger.error('Failed to initialize proxy manager', { error }); - this.isInitialized = true; // Set to true anyway to avoid infinite retries - } - } - - getProxy(): string | null { - if (this.proxies.length === 0) { - this.logger.warn('No proxies available in memory'); - return null; - } - - // Cycle through proxies - if (this.proxyIndex >= this.proxies.length) { - this.proxyIndex = 0; - } - - const proxyInfo = this.proxies[this.proxyIndex++]; - if (!proxyInfo) { - return null; - } - - // Build proxy URL with optional auth - let proxyUrl = `${proxyInfo.protocol}://`; - if (proxyInfo.username && proxyInfo.password) { - proxyUrl += `${proxyInfo.username}:${proxyInfo.password}@`; - } - proxyUrl += `${proxyInfo.host}:${proxyInfo.port}`; - - return proxyUrl; - } - /** - * Get a random working proxy from the available pool (synchronous) - */ - getRandomProxy(): ProxyInfo | null { - // Ensure initialized - if (!this.isInitialized) { - throw new Error('ProxyManager not initialized'); - } - - // Return null if no proxies available - if (this.proxies.length === 0) { - this.logger.warn('No proxies available in memory'); - return null; - } - - // Filter for working proxies (not explicitly marked as non-working) - const workingProxies = this.proxies.filter(proxy => proxy.isWorking !== false); - - if (workingProxies.length === 0) { - this.logger.warn('No working proxies available'); - return null; - } - - // Return random proxy with preference for recently successful ones - const sortedProxies = workingProxies.sort((a, b) => { - // Prefer proxies with better success rates - const aRate = a.successRate || 0; - const bRate = b.successRate || 0; - return bRate - aRate; - }); - - // Take from top 50% of best performing proxies - const topProxies = sortedProxies.slice(0, Math.max(1, Math.floor(sortedProxies.length * 0.5))); - const selectedProxy = topProxies[Math.floor(Math.random() * topProxies.length)]; - - if (!selectedProxy) { - this.logger.warn('No proxy selected from available pool'); - return null; - } - - this.logger.debug('Selected proxy', { - host: selectedProxy.host, - port: selectedProxy.port, - successRate: selectedProxy.successRate, - totalAvailable: workingProxies.length, - }); - - return selectedProxy; - } - - /** - * Get all working proxies (synchronous) - */ - getWorkingProxies(): ProxyInfo[] { - if (!this.isInitialized) { - throw new Error('ProxyManager not initialized'); - } - - return this.proxies.filter(proxy => proxy.isWorking !== false); - } - - /** - * Get all proxies (working and non-working) - */ - getAllProxies(): ProxyInfo[] { - if (!this.isInitialized) { - throw new Error('ProxyManager not initialized'); - } - - return [...this.proxies]; - } - - /** - * Get proxy statistics - */ - getStats(): ProxyStats { - if (!this.isInitialized) { - throw new Error('ProxyManager not initialized'); - } - - return { - total: this.proxies.length, - working: this.proxies.filter(p => p.isWorking !== false).length, - failed: this.proxies.filter(p => p.isWorking === false).length, - lastUpdate: this.lastUpdate - }; - } - - /** - * Update the proxy pool with new proxies - */ - async updateProxies(proxies: ProxyInfo[]): Promise { - // Ensure manager is initialized before updating - if (!this.isInitialized) { - await this.initializeInternal(); - } - - try { - this.logger.info('Updating proxy pool', { newCount: proxies.length, existingCount: this.proxies.length }); - - this.proxies = proxies; - this.lastUpdate = new Date(); - - // Store to cache - await this.cache.set('active-proxies', proxies); - await this.cache.set('last-update', this.lastUpdate.toISOString()); - - const workingCount = proxies.filter(p => p.isWorking !== false).length; - this.logger.info('Proxy pool updated successfully', { - totalProxies: proxies.length, - workingProxies: workingCount, - lastUpdate: this.lastUpdate, - }); - } catch (error) { - this.logger.error('Failed to update proxy pool', { error }); - throw error; - } - } - - /** - * Add or update a single proxy in the pool - */ - async updateProxy(proxy: ProxyInfo): Promise { - const existingIndex = this.proxies.findIndex( - p => p.host === proxy.host && p.port === proxy.port && p.protocol === proxy.protocol - ); - - if (existingIndex >= 0) { - this.proxies[existingIndex] = { ...this.proxies[existingIndex], ...proxy }; - this.logger.debug('Updated existing proxy', { host: proxy.host, port: proxy.port }); - } else { - this.proxies.push(proxy); - this.logger.debug('Added new proxy', { host: proxy.host, port: proxy.port }); - } - - // Update cache - await this.updateProxies(this.proxies); - } - - /** - * Remove a proxy from the pool - */ - async removeProxy(host: string, port: number, protocol: string): Promise { - const initialLength = this.proxies.length; - this.proxies = this.proxies.filter( - p => !(p.host === host && p.port === port && p.protocol === protocol) - ); - - if (this.proxies.length < initialLength) { - await this.updateProxies(this.proxies); - this.logger.debug('Removed proxy', { host, port, protocol }); - } - } - - /** - * Clear all proxies from memory and cache - */ - async clearProxies(): Promise { - this.proxies = []; - this.lastUpdate = null; - - await this.cache.del('active-proxies'); - await this.cache.del('last-update'); - - this.logger.info('Cleared all proxies'); - } - - /** - * Check if proxy manager is ready - */ - isReady(): boolean { - return this.isInitialized; - } - - /** - * Load proxies from cache storage - */ - private async loadFromCache(): Promise { - try { - const cachedProxies = await this.cache.get('active-proxies'); - const lastUpdateStr = await this.cache.get('last-update'); - - if (cachedProxies && Array.isArray(cachedProxies)) { - this.proxies = cachedProxies; - this.lastUpdate = lastUpdateStr ? new Date(lastUpdateStr) : null; - - this.logger.debug('Loaded proxies from cache', { - count: this.proxies.length, - lastUpdate: this.lastUpdate, - }); - } else { - this.logger.debug('No cached proxies found'); - } - } catch (error) { - this.logger.error('Failed to load proxies from cache', { error }); - } - } - - /** - * Initialize the proxy manager - */ - async initialize(): Promise { - await this.initializeInternal(); - - // Note: Initial proxy sync should be handled by the container or application - // that creates ProxyManager instance - this.logger.info('ProxyManager initialized - proxy sync should be handled externally'); - } -} - -// Export the class as default -export default ProxyManager; \ No newline at end of file +/** + * Centralized Proxy Manager - Handles proxy storage, retrieval, and caching + */ +import type { CacheProvider } from '@stock-bot/cache'; +import type { ProxyInfo, ProxyManagerConfig, ProxyStats } from './types'; + +export class ProxyManager { + private cache: CacheProvider; + private proxies: ProxyInfo[] = []; + private proxyIndex: number = 0; + private lastUpdate: Date | null = null; + private isInitialized = false; + private logger: any; + + constructor(cache: CacheProvider, _config: ProxyManagerConfig = {}, logger?: any) { + this.cache = cache; + this.logger = logger || console; + // Config can be used in the future for customization + } + + /** + * Internal initialization - loads existing proxies from cache + */ + private async initializeInternal(): Promise { + if (this.isInitialized) { + return; + } + + try { + this.logger.info('Initializing proxy manager...'); + + // Wait for cache to be ready + await this.cache.waitForReady(10000); // Wait up to 10 seconds + this.logger.debug('Cache is ready'); + + await this.loadFromCache(); + this.isInitialized = true; + this.logger.info('Proxy manager initialized', { + proxiesLoaded: this.proxies.length, + lastUpdate: this.lastUpdate, + }); + } catch (error) { + this.logger.error('Failed to initialize proxy manager', { error }); + this.isInitialized = true; // Set to true anyway to avoid infinite retries + } + } + + getProxy(): string | null { + if (this.proxies.length === 0) { + this.logger.warn('No proxies available in memory'); + return null; + } + + // Cycle through proxies + if (this.proxyIndex >= this.proxies.length) { + this.proxyIndex = 0; + } + + const proxyInfo = this.proxies[this.proxyIndex++]; + if (!proxyInfo) { + return null; + } + + // Build proxy URL with optional auth + let proxyUrl = `${proxyInfo.protocol}://`; + if (proxyInfo.username && proxyInfo.password) { + proxyUrl += `${proxyInfo.username}:${proxyInfo.password}@`; + } + proxyUrl += `${proxyInfo.host}:${proxyInfo.port}`; + + return proxyUrl; + } + /** + * Get a random working proxy from the available pool (synchronous) + */ + getRandomProxy(): ProxyInfo | null { + // Ensure initialized + if (!this.isInitialized) { + throw new Error('ProxyManager not initialized'); + } + + // Return null if no proxies available + if (this.proxies.length === 0) { + this.logger.warn('No proxies available in memory'); + return null; + } + + // Filter for working proxies (not explicitly marked as non-working) + const workingProxies = this.proxies.filter(proxy => proxy.isWorking !== false); + + if (workingProxies.length === 0) { + this.logger.warn('No working proxies available'); + return null; + } + + // Return random proxy with preference for recently successful ones + const sortedProxies = workingProxies.sort((a, b) => { + // Prefer proxies with better success rates + const aRate = a.successRate || 0; + const bRate = b.successRate || 0; + return bRate - aRate; + }); + + // Take from top 50% of best performing proxies + const topProxies = sortedProxies.slice(0, Math.max(1, Math.floor(sortedProxies.length * 0.5))); + const selectedProxy = topProxies[Math.floor(Math.random() * topProxies.length)]; + + if (!selectedProxy) { + this.logger.warn('No proxy selected from available pool'); + return null; + } + + this.logger.debug('Selected proxy', { + host: selectedProxy.host, + port: selectedProxy.port, + successRate: selectedProxy.successRate, + totalAvailable: workingProxies.length, + }); + + return selectedProxy; + } + + /** + * Get all working proxies (synchronous) + */ + getWorkingProxies(): ProxyInfo[] { + if (!this.isInitialized) { + throw new Error('ProxyManager not initialized'); + } + + return this.proxies.filter(proxy => proxy.isWorking !== false); + } + + /** + * Get all proxies (working and non-working) + */ + getAllProxies(): ProxyInfo[] { + if (!this.isInitialized) { + throw new Error('ProxyManager not initialized'); + } + + return [...this.proxies]; + } + + /** + * Get proxy statistics + */ + getStats(): ProxyStats { + if (!this.isInitialized) { + throw new Error('ProxyManager not initialized'); + } + + return { + total: this.proxies.length, + working: this.proxies.filter(p => p.isWorking !== false).length, + failed: this.proxies.filter(p => p.isWorking === false).length, + lastUpdate: this.lastUpdate, + }; + } + + /** + * Update the proxy pool with new proxies + */ + async updateProxies(proxies: ProxyInfo[]): Promise { + // Ensure manager is initialized before updating + if (!this.isInitialized) { + await this.initializeInternal(); + } + + try { + this.logger.info('Updating proxy pool', { + newCount: proxies.length, + existingCount: this.proxies.length, + }); + + this.proxies = proxies; + this.lastUpdate = new Date(); + + // Store to cache + await this.cache.set('active-proxies', proxies); + await this.cache.set('last-update', this.lastUpdate.toISOString()); + + const workingCount = proxies.filter(p => p.isWorking !== false).length; + this.logger.info('Proxy pool updated successfully', { + totalProxies: proxies.length, + workingProxies: workingCount, + lastUpdate: this.lastUpdate, + }); + } catch (error) { + this.logger.error('Failed to update proxy pool', { error }); + throw error; + } + } + + /** + * Add or update a single proxy in the pool + */ + async updateProxy(proxy: ProxyInfo): Promise { + const existingIndex = this.proxies.findIndex( + p => p.host === proxy.host && p.port === proxy.port && p.protocol === proxy.protocol + ); + + if (existingIndex >= 0) { + this.proxies[existingIndex] = { ...this.proxies[existingIndex], ...proxy }; + this.logger.debug('Updated existing proxy', { host: proxy.host, port: proxy.port }); + } else { + this.proxies.push(proxy); + this.logger.debug('Added new proxy', { host: proxy.host, port: proxy.port }); + } + + // Update cache + await this.updateProxies(this.proxies); + } + + /** + * Remove a proxy from the pool + */ + async removeProxy(host: string, port: number, protocol: string): Promise { + const initialLength = this.proxies.length; + this.proxies = this.proxies.filter( + p => !(p.host === host && p.port === port && p.protocol === protocol) + ); + + if (this.proxies.length < initialLength) { + await this.updateProxies(this.proxies); + this.logger.debug('Removed proxy', { host, port, protocol }); + } + } + + /** + * Clear all proxies from memory and cache + */ + async clearProxies(): Promise { + this.proxies = []; + this.lastUpdate = null; + + await this.cache.del('active-proxies'); + await this.cache.del('last-update'); + + this.logger.info('Cleared all proxies'); + } + + /** + * Check if proxy manager is ready + */ + isReady(): boolean { + return this.isInitialized; + } + + /** + * Load proxies from cache storage + */ + private async loadFromCache(): Promise { + try { + const cachedProxies = await this.cache.get('active-proxies'); + const lastUpdateStr = await this.cache.get('last-update'); + + if (cachedProxies && Array.isArray(cachedProxies)) { + this.proxies = cachedProxies; + this.lastUpdate = lastUpdateStr ? new Date(lastUpdateStr) : null; + + this.logger.debug('Loaded proxies from cache', { + count: this.proxies.length, + lastUpdate: this.lastUpdate, + }); + } else { + this.logger.debug('No cached proxies found'); + } + } catch (error) { + this.logger.error('Failed to load proxies from cache', { error }); + } + } + + /** + * Initialize the proxy manager + */ + async initialize(): Promise { + await this.initializeInternal(); + + // Note: Initial proxy sync should be handled by the container or application + // that creates ProxyManager instance + this.logger.info('ProxyManager initialized - proxy sync should be handled externally'); + } +} + +// Export the class as default +export default ProxyManager; diff --git a/libs/services/proxy/src/types.ts b/libs/services/proxy/src/types.ts index c36b96e..963310f 100644 --- a/libs/services/proxy/src/types.ts +++ b/libs/services/proxy/src/types.ts @@ -1,42 +1,42 @@ -/** - * Proxy service types and interfaces - */ - -export interface ProxyInfo { - host: string; - port: number; - protocol: 'http' | 'https'; // Simplified to only support HTTP/HTTPS - username?: string; - password?: string; - isWorking?: boolean; - successRate?: number; - lastChecked?: Date; - lastUsed?: Date; - responseTime?: number; - source?: string; - country?: string; - error?: string; - // Tracking properties - working?: number; // Number of successful checks - total?: number; // Total number of checks - averageResponseTime?: number; // Average response time in milliseconds - firstSeen?: Date; // When the proxy was first added -} - -export interface ProxyManagerConfig { - cachePrefix?: string; - ttl?: number; - enableMetrics?: boolean; -} - -export interface ProxySyncConfig { - intervalMs?: number; - enableAutoSync?: boolean; -} - -export interface ProxyStats { - total: number; - working: number; - failed: number; - lastUpdate: Date | null; -} \ No newline at end of file +/** + * Proxy service types and interfaces + */ + +export interface ProxyInfo { + host: string; + port: number; + protocol: 'http' | 'https'; // Simplified to only support HTTP/HTTPS + username?: string; + password?: string; + isWorking?: boolean; + successRate?: number; + lastChecked?: Date; + lastUsed?: Date; + responseTime?: number; + source?: string; + country?: string; + error?: string; + // Tracking properties + working?: number; // Number of successful checks + total?: number; // Total number of checks + averageResponseTime?: number; // Average response time in milliseconds + firstSeen?: Date; // When the proxy was first added +} + +export interface ProxyManagerConfig { + cachePrefix?: string; + ttl?: number; + enableMetrics?: boolean; +} + +export interface ProxySyncConfig { + intervalMs?: number; + enableAutoSync?: boolean; +} + +export interface ProxyStats { + total: number; + working: number; + failed: number; + lastUpdate: Date | null; +} diff --git a/libs/services/proxy/tsconfig.json b/libs/services/proxy/tsconfig.json index ad33e78..0c67432 100644 --- a/libs/services/proxy/tsconfig.json +++ b/libs/services/proxy/tsconfig.json @@ -1,12 +1,12 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src", - "declaration": true, - "declarationMap": true, - "sourceMap": true - }, - "include": ["src/**/*"], - "exclude": ["dist", "node_modules"] -} \ No newline at end of file +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src", + "declaration": true, + "declarationMap": true, + "sourceMap": true + }, + "include": ["src/**/*"], + "exclude": ["dist", "node_modules"] +} diff --git a/libs/services/queue/src/dlq-handler.ts b/libs/services/queue/src/dlq-handler.ts index 640f69b..1e8abc1 100644 --- a/libs/services/queue/src/dlq-handler.ts +++ b/libs/services/queue/src/dlq-handler.ts @@ -1,251 +1,249 @@ -import { getLogger } from '@stock-bot/logger'; -import { Queue, type Job } from 'bullmq'; -import type { DLQConfig, RedisConfig } from './types'; -import { getRedisConnection } from './utils'; - -const logger = getLogger('dlq-handler'); - -export class DeadLetterQueueHandler { - private dlq: Queue; - private config: Required; - private failureCount = new Map(); - - constructor( - private mainQueue: Queue, - connection: RedisConfig, - config: DLQConfig = {} - ) { - this.config = { - maxRetries: config.maxRetries ?? 3, - retryDelay: config.retryDelay ?? 60000, // 1 minute - alertThreshold: config.alertThreshold ?? 100, - cleanupAge: config.cleanupAge ?? 168, // 7 days - }; - - // Create DLQ with same name but -dlq suffix - const dlqName = `${mainQueue.name}-dlq`; - this.dlq = new Queue(dlqName, { connection: getRedisConnection(connection) }); - } - - /** - * Process a failed job - either retry or move to DLQ - */ - async handleFailedJob(job: Job, error: Error): Promise { - const jobKey = `${job.name}:${job.id}`; - const currentFailures = (this.failureCount.get(jobKey) || 0) + 1; - this.failureCount.set(jobKey, currentFailures); - - logger.warn('Job failed', { - jobId: job.id, - jobName: job.name, - attempt: job.attemptsMade, - maxAttempts: job.opts.attempts, - error: error.message, - failureCount: currentFailures, - }); - - // Check if job should be moved to DLQ - if (job.attemptsMade >= (job.opts.attempts || this.config.maxRetries)) { - await this.moveToDeadLetterQueue(job, error); - this.failureCount.delete(jobKey); - } - } - - /** - * Move job to dead letter queue - */ - private async moveToDeadLetterQueue(job: Job, error: Error): Promise { - try { - const dlqData = { - originalJob: { - id: job.id, - name: job.name, - data: job.data, - opts: job.opts, - attemptsMade: job.attemptsMade, - failedReason: job.failedReason, - processedOn: job.processedOn, - timestamp: job.timestamp, - }, - error: { - message: error.message, - stack: error.stack, - name: error.name, - }, - movedToDLQAt: new Date().toISOString(), - }; - - await this.dlq.add('failed-job', dlqData, { - removeOnComplete: 100, - removeOnFail: 50, - }); - - logger.error('Job moved to DLQ', { - jobId: job.id, - jobName: job.name, - error: error.message, - }); - - // Check if we need to alert - await this.checkAlertThreshold(); - } catch (dlqError) { - logger.error('Failed to move job to DLQ', { - jobId: job.id, - error: dlqError, - }); - } - } - - /** - * Retry jobs from DLQ - */ - async retryDLQJobs(limit = 10): Promise { - const jobs = await this.dlq.getCompleted(0, limit); - let retriedCount = 0; - - for (const dlqJob of jobs) { - try { - const { originalJob } = dlqJob.data; - - // Re-add to main queue with delay - await this.mainQueue.add( - originalJob.name, - originalJob.data, - { - ...originalJob.opts, - delay: this.config.retryDelay, - attempts: this.config.maxRetries, - } - ); - - // Remove from DLQ - await dlqJob.remove(); - retriedCount++; - - logger.info('Job retried from DLQ', { - originalJobId: originalJob.id, - jobName: originalJob.name, - }); - } catch (error) { - logger.error('Failed to retry DLQ job', { - dlqJobId: dlqJob.id, - error, - }); - } - } - - return retriedCount; - } - - /** - * Get DLQ statistics - */ - async getStats(): Promise<{ - total: number; - recent: number; - byJobName: Record; - oldestJob: Date | null; - }> { - const [completed, failed, waiting] = await Promise.all([ - this.dlq.getCompleted(), - this.dlq.getFailed(), - this.dlq.getWaiting(), - ]); - - const allJobs = [...completed, ...failed, ...waiting]; - const byJobName: Record = {}; - let oldestTimestamp: number | null = null; - - for (const job of allJobs) { - const jobName = job.data.originalJob?.name || 'unknown'; - byJobName[jobName] = (byJobName[jobName] || 0) + 1; - - if (!oldestTimestamp || job.timestamp < oldestTimestamp) { - oldestTimestamp = job.timestamp; - } - } - - // Count recent jobs (last 24 hours) - const oneDayAgo = Date.now() - 24 * 60 * 60 * 1000; - const recent = allJobs.filter(job => job.timestamp > oneDayAgo).length; - - return { - total: allJobs.length, - recent, - byJobName, - oldestJob: oldestTimestamp ? new Date(oldestTimestamp) : null, - }; - } - - /** - * Clean up old DLQ entries - */ - async cleanup(): Promise { - const ageInMs = this.config.cleanupAge * 60 * 60 * 1000; - const cutoffTime = Date.now() - ageInMs; - - const jobs = await this.dlq.getCompleted(); - let removedCount = 0; - - for (const job of jobs) { - if (job.timestamp < cutoffTime) { - await job.remove(); - removedCount++; - } - } - - logger.info('DLQ cleanup completed', { - removedCount, - cleanupAge: `${this.config.cleanupAge} hours`, - }); - - return removedCount; - } - - /** - * Check if alert threshold is exceeded - */ - private async checkAlertThreshold(): Promise { - const stats = await this.getStats(); - - if (stats.total >= this.config.alertThreshold) { - logger.error('DLQ alert threshold exceeded', { - threshold: this.config.alertThreshold, - currentCount: stats.total, - byJobName: stats.byJobName, - }); - // In a real implementation, this would trigger alerts - } - } - - /** - * Get failed jobs for inspection - */ - async inspectFailedJobs(limit = 10): Promise> { - const jobs = await this.dlq.getCompleted(0, limit); - - return jobs.map(job => ({ - id: job.data.originalJob.id, - name: job.data.originalJob.name, - data: job.data.originalJob.data, - error: job.data.error, - failedAt: job.data.movedToDLQAt, - attempts: job.data.originalJob.attemptsMade, - })); - } - - /** - * Shutdown DLQ handler - */ - async shutdown(): Promise { - await this.dlq.close(); - this.failureCount.clear(); - } -} \ No newline at end of file +import { Queue, type Job } from 'bullmq'; +import { getLogger } from '@stock-bot/logger'; +import type { DLQConfig, RedisConfig } from './types'; +import { getRedisConnection } from './utils'; + +const logger = getLogger('dlq-handler'); + +export class DeadLetterQueueHandler { + private dlq: Queue; + private config: Required; + private failureCount = new Map(); + + constructor( + private mainQueue: Queue, + connection: RedisConfig, + config: DLQConfig = {} + ) { + this.config = { + maxRetries: config.maxRetries ?? 3, + retryDelay: config.retryDelay ?? 60000, // 1 minute + alertThreshold: config.alertThreshold ?? 100, + cleanupAge: config.cleanupAge ?? 168, // 7 days + }; + + // Create DLQ with same name but -dlq suffix + const dlqName = `${mainQueue.name}-dlq`; + this.dlq = new Queue(dlqName, { connection: getRedisConnection(connection) }); + } + + /** + * Process a failed job - either retry or move to DLQ + */ + async handleFailedJob(job: Job, error: Error): Promise { + const jobKey = `${job.name}:${job.id}`; + const currentFailures = (this.failureCount.get(jobKey) || 0) + 1; + this.failureCount.set(jobKey, currentFailures); + + logger.warn('Job failed', { + jobId: job.id, + jobName: job.name, + attempt: job.attemptsMade, + maxAttempts: job.opts.attempts, + error: error.message, + failureCount: currentFailures, + }); + + // Check if job should be moved to DLQ + if (job.attemptsMade >= (job.opts.attempts || this.config.maxRetries)) { + await this.moveToDeadLetterQueue(job, error); + this.failureCount.delete(jobKey); + } + } + + /** + * Move job to dead letter queue + */ + private async moveToDeadLetterQueue(job: Job, error: Error): Promise { + try { + const dlqData = { + originalJob: { + id: job.id, + name: job.name, + data: job.data, + opts: job.opts, + attemptsMade: job.attemptsMade, + failedReason: job.failedReason, + processedOn: job.processedOn, + timestamp: job.timestamp, + }, + error: { + message: error.message, + stack: error.stack, + name: error.name, + }, + movedToDLQAt: new Date().toISOString(), + }; + + await this.dlq.add('failed-job', dlqData, { + removeOnComplete: 100, + removeOnFail: 50, + }); + + logger.error('Job moved to DLQ', { + jobId: job.id, + jobName: job.name, + error: error.message, + }); + + // Check if we need to alert + await this.checkAlertThreshold(); + } catch (dlqError) { + logger.error('Failed to move job to DLQ', { + jobId: job.id, + error: dlqError, + }); + } + } + + /** + * Retry jobs from DLQ + */ + async retryDLQJobs(limit = 10): Promise { + const jobs = await this.dlq.getCompleted(0, limit); + let retriedCount = 0; + + for (const dlqJob of jobs) { + try { + const { originalJob } = dlqJob.data; + + // Re-add to main queue with delay + await this.mainQueue.add(originalJob.name, originalJob.data, { + ...originalJob.opts, + delay: this.config.retryDelay, + attempts: this.config.maxRetries, + }); + + // Remove from DLQ + await dlqJob.remove(); + retriedCount++; + + logger.info('Job retried from DLQ', { + originalJobId: originalJob.id, + jobName: originalJob.name, + }); + } catch (error) { + logger.error('Failed to retry DLQ job', { + dlqJobId: dlqJob.id, + error, + }); + } + } + + return retriedCount; + } + + /** + * Get DLQ statistics + */ + async getStats(): Promise<{ + total: number; + recent: number; + byJobName: Record; + oldestJob: Date | null; + }> { + const [completed, failed, waiting] = await Promise.all([ + this.dlq.getCompleted(), + this.dlq.getFailed(), + this.dlq.getWaiting(), + ]); + + const allJobs = [...completed, ...failed, ...waiting]; + const byJobName: Record = {}; + let oldestTimestamp: number | null = null; + + for (const job of allJobs) { + const jobName = job.data.originalJob?.name || 'unknown'; + byJobName[jobName] = (byJobName[jobName] || 0) + 1; + + if (!oldestTimestamp || job.timestamp < oldestTimestamp) { + oldestTimestamp = job.timestamp; + } + } + + // Count recent jobs (last 24 hours) + const oneDayAgo = Date.now() - 24 * 60 * 60 * 1000; + const recent = allJobs.filter(job => job.timestamp > oneDayAgo).length; + + return { + total: allJobs.length, + recent, + byJobName, + oldestJob: oldestTimestamp ? new Date(oldestTimestamp) : null, + }; + } + + /** + * Clean up old DLQ entries + */ + async cleanup(): Promise { + const ageInMs = this.config.cleanupAge * 60 * 60 * 1000; + const cutoffTime = Date.now() - ageInMs; + + const jobs = await this.dlq.getCompleted(); + let removedCount = 0; + + for (const job of jobs) { + if (job.timestamp < cutoffTime) { + await job.remove(); + removedCount++; + } + } + + logger.info('DLQ cleanup completed', { + removedCount, + cleanupAge: `${this.config.cleanupAge} hours`, + }); + + return removedCount; + } + + /** + * Check if alert threshold is exceeded + */ + private async checkAlertThreshold(): Promise { + const stats = await this.getStats(); + + if (stats.total >= this.config.alertThreshold) { + logger.error('DLQ alert threshold exceeded', { + threshold: this.config.alertThreshold, + currentCount: stats.total, + byJobName: stats.byJobName, + }); + // In a real implementation, this would trigger alerts + } + } + + /** + * Get failed jobs for inspection + */ + async inspectFailedJobs(limit = 10): Promise< + Array<{ + id: string; + name: string; + data: unknown; + error: unknown; + failedAt: string; + attempts: number; + }> + > { + const jobs = await this.dlq.getCompleted(0, limit); + + return jobs.map(job => ({ + id: job.data.originalJob.id, + name: job.data.originalJob.name, + data: job.data.originalJob.data, + error: job.data.error, + failedAt: job.data.movedToDLQAt, + attempts: job.data.originalJob.attemptsMade, + })); + } + + /** + * Shutdown DLQ handler + */ + async shutdown(): Promise { + await this.dlq.close(); + this.failureCount.clear(); + } +} diff --git a/libs/services/queue/src/index.ts b/libs/services/queue/src/index.ts index f6224aa..6072409 100644 --- a/libs/services/queue/src/index.ts +++ b/libs/services/queue/src/index.ts @@ -26,34 +26,33 @@ export type { QueueOptions, QueueStats, GlobalStats, - + // Batch processing types BatchResult, ProcessOptions, BatchJobData, - + // Handler types JobHandler, TypedJobHandler, HandlerConfig, HandlerConfigWithSchedule, HandlerInitializer, - + // Configuration types RedisConfig, QueueConfig, QueueManagerConfig, - + // Rate limiting types RateLimitConfig, RateLimitRule, - + // DLQ types DLQConfig, DLQJobInfo, - + // Scheduled job types ScheduledJob, ScheduleConfig, } from './types'; - diff --git a/libs/services/queue/src/queue-manager.ts b/libs/services/queue/src/queue-manager.ts index de5c45c..620f93b 100644 --- a/libs/services/queue/src/queue-manager.ts +++ b/libs/services/queue/src/queue-manager.ts @@ -130,7 +130,8 @@ export class QueueManager { const queueConfig: QueueWorkerConfig = { workers: mergedOptions.workers, concurrency: mergedOptions.concurrency, - startWorker: !!mergedOptions.workers && mergedOptions.workers > 0 && !this.config.delayWorkerStart, + startWorker: + !!mergedOptions.workers && mergedOptions.workers > 0 && !this.config.delayWorkerStart, }; const queue = new Queue( @@ -443,7 +444,9 @@ export class QueueManager { */ startAllWorkers(): void { if (!this.config.delayWorkerStart) { - logger.info('startAllWorkers() called but workers already started automatically (delayWorkerStart is false)'); + logger.info( + 'startAllWorkers() called but workers already started automatically (delayWorkerStart is false)' + ); return; } @@ -451,17 +454,17 @@ export class QueueManager { for (const queue of this.queues.values()) { const workerCount = this.config.defaultQueueOptions?.workers || 1; const concurrency = this.config.defaultQueueOptions?.concurrency || 1; - + if (workerCount > 0) { queue.startWorkersManually(workerCount, concurrency); workersStarted++; } } - logger.info('All workers started', { + logger.info('All workers started', { totalQueues: this.queues.size, queuesWithWorkers: workersStarted, - delayWorkerStart: this.config.delayWorkerStart + delayWorkerStart: this.config.delayWorkerStart, }); } diff --git a/libs/services/queue/src/queue-metrics.ts b/libs/services/queue/src/queue-metrics.ts index c74ada5..e45477d 100644 --- a/libs/services/queue/src/queue-metrics.ts +++ b/libs/services/queue/src/queue-metrics.ts @@ -1,314 +1,318 @@ -import { Queue, QueueEvents } from 'bullmq'; -// import { getLogger } from '@stock-bot/logger'; - -// const logger = getLogger('queue-metrics'); - -export interface QueueMetrics { - // Job counts - waiting: number; - active: number; - completed: number; - failed: number; - delayed: number; - paused?: number; - - // Performance metrics - processingTime: { - avg: number; - min: number; - max: number; - p95: number; - p99: number; - }; - - // Throughput - throughput: { - completedPerMinute: number; - failedPerMinute: number; - totalPerMinute: number; - }; - - // Job age - oldestWaitingJob: Date | null; - - // Health - isHealthy: boolean; - healthIssues: string[]; -} - -export class QueueMetricsCollector { - private processingTimes: number[] = []; - private completedTimestamps: number[] = []; - private failedTimestamps: number[] = []; - private jobStartTimes = new Map(); - private readonly maxSamples = 1000; - private readonly metricsInterval = 60000; // 1 minute - - constructor( - private queue: Queue, - private queueEvents: QueueEvents - ) { - this.setupEventListeners(); - } - - /** - * Setup event listeners for metrics collection - */ - private setupEventListeners(): void { - this.queueEvents.on('completed', () => { - // Record completion - this.completedTimestamps.push(Date.now()); - this.cleanupOldTimestamps(); - }); - - this.queueEvents.on('failed', () => { - // Record failure - this.failedTimestamps.push(Date.now()); - this.cleanupOldTimestamps(); - }); - - // Track processing times - this.queueEvents.on('active', ({ jobId }) => { - this.jobStartTimes.set(jobId, Date.now()); - }); - - this.queueEvents.on('completed', ({ jobId }) => { - const startTime = this.jobStartTimes.get(jobId); - if (startTime) { - const processingTime = Date.now() - startTime; - this.recordProcessingTime(processingTime); - this.jobStartTimes.delete(jobId); - } - }); - } - - /** - * Record processing time - */ - private recordProcessingTime(time: number): void { - this.processingTimes.push(time); - - // Keep only recent samples - if (this.processingTimes.length > this.maxSamples) { - this.processingTimes = this.processingTimes.slice(-this.maxSamples); - } - } - - /** - * Clean up old timestamps - */ - private cleanupOldTimestamps(): void { - const cutoff = Date.now() - this.metricsInterval; - - this.completedTimestamps = this.completedTimestamps.filter(ts => ts > cutoff); - this.failedTimestamps = this.failedTimestamps.filter(ts => ts > cutoff); - } - - /** - * Collect current metrics - */ - async collect(): Promise { - // Get job counts - const [waiting, active, completed, failed, delayed] = await Promise.all([ - this.queue.getWaitingCount(), - this.queue.getActiveCount(), - this.queue.getCompletedCount(), - this.queue.getFailedCount(), - this.queue.getDelayedCount(), - ]); - - // BullMQ doesn't have getPausedCount, check if queue is paused - const paused = await this.queue.isPaused() ? waiting : 0; - - // Calculate processing time metrics - const processingTime = this.calculateProcessingTimeMetrics(); - - // Calculate throughput - const throughput = this.calculateThroughput(); - - // Get oldest waiting job - const oldestWaitingJob = await this.getOldestWaitingJob(); - - // Check health - const { isHealthy, healthIssues } = this.checkHealth({ - waiting, - active, - failed, - processingTime, - }); - - return { - waiting, - active, - completed, - failed, - delayed, - paused, - processingTime, - throughput, - oldestWaitingJob, - isHealthy, - healthIssues, - }; - } - - /** - * Calculate processing time metrics - */ - private calculateProcessingTimeMetrics(): QueueMetrics['processingTime'] { - if (this.processingTimes.length === 0) { - return { avg: 0, min: 0, max: 0, p95: 0, p99: 0 }; - } - - const sorted = [...this.processingTimes].sort((a, b) => a - b); - const sum = sorted.reduce((acc, val) => acc + val, 0); - - return { - avg: sorted.length > 0 ? Math.round(sum / sorted.length) : 0, - min: sorted[0] || 0, - max: sorted[sorted.length - 1] || 0, - p95: sorted[Math.floor(sorted.length * 0.95)] || 0, - p99: sorted[Math.floor(sorted.length * 0.99)] || 0, - }; - } - - /** - * Calculate throughput metrics - */ - private calculateThroughput(): QueueMetrics['throughput'] { - const now = Date.now(); - const oneMinuteAgo = now - 60000; - - const completedPerMinute = this.completedTimestamps.filter(ts => ts > oneMinuteAgo).length; - const failedPerMinute = this.failedTimestamps.filter(ts => ts > oneMinuteAgo).length; - - return { - completedPerMinute, - failedPerMinute, - totalPerMinute: completedPerMinute + failedPerMinute, - }; - } - - /** - * Get oldest waiting job - */ - private async getOldestWaitingJob(): Promise { - const waitingJobs = await this.queue.getWaiting(0, 1); - - if (waitingJobs.length > 0) { - return new Date(waitingJobs[0].timestamp); - } - - return null; - } - - /** - * Check queue health - */ - private checkHealth(metrics: { - waiting: number; - active: number; - failed: number; - processingTime: QueueMetrics['processingTime']; - }): { isHealthy: boolean; healthIssues: string[] } { - const issues: string[] = []; - - // Check for high failure rate - const failureRate = metrics.failed / (metrics.failed + this.completedTimestamps.length); - if (failureRate > 0.1) { - issues.push(`High failure rate: ${(failureRate * 100).toFixed(1)}%`); - } - - // Check for queue backlog - if (metrics.waiting > 1000) { - issues.push(`Large queue backlog: ${metrics.waiting} jobs waiting`); - } - - // Check for slow processing - if (metrics.processingTime.avg > 30000) { // 30 seconds - issues.push(`Slow average processing time: ${(metrics.processingTime.avg / 1000).toFixed(1)}s`); - } - - // Check for stalled active jobs - if (metrics.active > 100) { - issues.push(`High number of active jobs: ${metrics.active}`); - } - - return { - isHealthy: issues.length === 0, - healthIssues: issues, - }; - } - - /** - * Get formatted metrics report - */ - async getReport(): Promise { - const metrics = await this.collect(); - - return ` -Queue Metrics Report -=================== -Status: ${metrics.isHealthy ? '✅ Healthy' : '⚠️ Issues Detected'} - -Job Counts: -- Waiting: ${metrics.waiting} -- Active: ${metrics.active} -- Completed: ${metrics.completed} -- Failed: ${metrics.failed} -- Delayed: ${metrics.delayed} -- Paused: ${metrics.paused} - -Performance: -- Avg Processing Time: ${(metrics.processingTime.avg / 1000).toFixed(2)}s -- Min/Max: ${(metrics.processingTime.min / 1000).toFixed(2)}s / ${(metrics.processingTime.max / 1000).toFixed(2)}s -- P95/P99: ${(metrics.processingTime.p95 / 1000).toFixed(2)}s / ${(metrics.processingTime.p99 / 1000).toFixed(2)}s - -Throughput: -- Completed/min: ${metrics.throughput.completedPerMinute} -- Failed/min: ${metrics.throughput.failedPerMinute} -- Total/min: ${metrics.throughput.totalPerMinute} - -${metrics.oldestWaitingJob ? `Oldest Waiting Job: ${metrics.oldestWaitingJob.toISOString()}` : 'No waiting jobs'} - -${metrics.healthIssues.length > 0 ? `\nHealth Issues:\n${metrics.healthIssues.map(issue => `- ${issue}`).join('\n')}` : ''} - `.trim(); - } - - /** - * Export metrics in Prometheus format - */ - async getPrometheusMetrics(): Promise { - const metrics = await this.collect(); - const queueName = this.queue.name; - - return ` -# HELP queue_jobs_total Total number of jobs by status -# TYPE queue_jobs_total gauge -queue_jobs_total{queue="${queueName}",status="waiting"} ${metrics.waiting} -queue_jobs_total{queue="${queueName}",status="active"} ${metrics.active} -queue_jobs_total{queue="${queueName}",status="completed"} ${metrics.completed} -queue_jobs_total{queue="${queueName}",status="failed"} ${metrics.failed} -queue_jobs_total{queue="${queueName}",status="delayed"} ${metrics.delayed} -queue_jobs_total{queue="${queueName}",status="paused"} ${metrics.paused} - -# HELP queue_processing_time_seconds Job processing time in seconds -# TYPE queue_processing_time_seconds summary -queue_processing_time_seconds{queue="${queueName}",quantile="0.5"} ${(metrics.processingTime.avg / 1000).toFixed(3)} -queue_processing_time_seconds{queue="${queueName}",quantile="0.95"} ${(metrics.processingTime.p95 / 1000).toFixed(3)} -queue_processing_time_seconds{queue="${queueName}",quantile="0.99"} ${(metrics.processingTime.p99 / 1000).toFixed(3)} -queue_processing_time_seconds_sum{queue="${queueName}"} ${(metrics.processingTime.avg * this.processingTimes.length / 1000).toFixed(3)} -queue_processing_time_seconds_count{queue="${queueName}"} ${this.processingTimes.length} - -# HELP queue_throughput_per_minute Jobs processed per minute -# TYPE queue_throughput_per_minute gauge -queue_throughput_per_minute{queue="${queueName}",status="completed"} ${metrics.throughput.completedPerMinute} -queue_throughput_per_minute{queue="${queueName}",status="failed"} ${metrics.throughput.failedPerMinute} -queue_throughput_per_minute{queue="${queueName}",status="total"} ${metrics.throughput.totalPerMinute} - -# HELP queue_health Queue health status -# TYPE queue_health gauge -queue_health{queue="${queueName}"} ${metrics.isHealthy ? 1 : 0} - `.trim(); - } -} \ No newline at end of file +import { Queue, QueueEvents } from 'bullmq'; + +// import { getLogger } from '@stock-bot/logger'; + +// const logger = getLogger('queue-metrics'); + +export interface QueueMetrics { + // Job counts + waiting: number; + active: number; + completed: number; + failed: number; + delayed: number; + paused?: number; + + // Performance metrics + processingTime: { + avg: number; + min: number; + max: number; + p95: number; + p99: number; + }; + + // Throughput + throughput: { + completedPerMinute: number; + failedPerMinute: number; + totalPerMinute: number; + }; + + // Job age + oldestWaitingJob: Date | null; + + // Health + isHealthy: boolean; + healthIssues: string[]; +} + +export class QueueMetricsCollector { + private processingTimes: number[] = []; + private completedTimestamps: number[] = []; + private failedTimestamps: number[] = []; + private jobStartTimes = new Map(); + private readonly maxSamples = 1000; + private readonly metricsInterval = 60000; // 1 minute + + constructor( + private queue: Queue, + private queueEvents: QueueEvents + ) { + this.setupEventListeners(); + } + + /** + * Setup event listeners for metrics collection + */ + private setupEventListeners(): void { + this.queueEvents.on('completed', () => { + // Record completion + this.completedTimestamps.push(Date.now()); + this.cleanupOldTimestamps(); + }); + + this.queueEvents.on('failed', () => { + // Record failure + this.failedTimestamps.push(Date.now()); + this.cleanupOldTimestamps(); + }); + + // Track processing times + this.queueEvents.on('active', ({ jobId }) => { + this.jobStartTimes.set(jobId, Date.now()); + }); + + this.queueEvents.on('completed', ({ jobId }) => { + const startTime = this.jobStartTimes.get(jobId); + if (startTime) { + const processingTime = Date.now() - startTime; + this.recordProcessingTime(processingTime); + this.jobStartTimes.delete(jobId); + } + }); + } + + /** + * Record processing time + */ + private recordProcessingTime(time: number): void { + this.processingTimes.push(time); + + // Keep only recent samples + if (this.processingTimes.length > this.maxSamples) { + this.processingTimes = this.processingTimes.slice(-this.maxSamples); + } + } + + /** + * Clean up old timestamps + */ + private cleanupOldTimestamps(): void { + const cutoff = Date.now() - this.metricsInterval; + + this.completedTimestamps = this.completedTimestamps.filter(ts => ts > cutoff); + this.failedTimestamps = this.failedTimestamps.filter(ts => ts > cutoff); + } + + /** + * Collect current metrics + */ + async collect(): Promise { + // Get job counts + const [waiting, active, completed, failed, delayed] = await Promise.all([ + this.queue.getWaitingCount(), + this.queue.getActiveCount(), + this.queue.getCompletedCount(), + this.queue.getFailedCount(), + this.queue.getDelayedCount(), + ]); + + // BullMQ doesn't have getPausedCount, check if queue is paused + const paused = (await this.queue.isPaused()) ? waiting : 0; + + // Calculate processing time metrics + const processingTime = this.calculateProcessingTimeMetrics(); + + // Calculate throughput + const throughput = this.calculateThroughput(); + + // Get oldest waiting job + const oldestWaitingJob = await this.getOldestWaitingJob(); + + // Check health + const { isHealthy, healthIssues } = this.checkHealth({ + waiting, + active, + failed, + processingTime, + }); + + return { + waiting, + active, + completed, + failed, + delayed, + paused, + processingTime, + throughput, + oldestWaitingJob, + isHealthy, + healthIssues, + }; + } + + /** + * Calculate processing time metrics + */ + private calculateProcessingTimeMetrics(): QueueMetrics['processingTime'] { + if (this.processingTimes.length === 0) { + return { avg: 0, min: 0, max: 0, p95: 0, p99: 0 }; + } + + const sorted = [...this.processingTimes].sort((a, b) => a - b); + const sum = sorted.reduce((acc, val) => acc + val, 0); + + return { + avg: sorted.length > 0 ? Math.round(sum / sorted.length) : 0, + min: sorted[0] || 0, + max: sorted[sorted.length - 1] || 0, + p95: sorted[Math.floor(sorted.length * 0.95)] || 0, + p99: sorted[Math.floor(sorted.length * 0.99)] || 0, + }; + } + + /** + * Calculate throughput metrics + */ + private calculateThroughput(): QueueMetrics['throughput'] { + const now = Date.now(); + const oneMinuteAgo = now - 60000; + + const completedPerMinute = this.completedTimestamps.filter(ts => ts > oneMinuteAgo).length; + const failedPerMinute = this.failedTimestamps.filter(ts => ts > oneMinuteAgo).length; + + return { + completedPerMinute, + failedPerMinute, + totalPerMinute: completedPerMinute + failedPerMinute, + }; + } + + /** + * Get oldest waiting job + */ + private async getOldestWaitingJob(): Promise { + const waitingJobs = await this.queue.getWaiting(0, 1); + + if (waitingJobs.length > 0) { + return new Date(waitingJobs[0].timestamp); + } + + return null; + } + + /** + * Check queue health + */ + private checkHealth(metrics: { + waiting: number; + active: number; + failed: number; + processingTime: QueueMetrics['processingTime']; + }): { isHealthy: boolean; healthIssues: string[] } { + const issues: string[] = []; + + // Check for high failure rate + const failureRate = metrics.failed / (metrics.failed + this.completedTimestamps.length); + if (failureRate > 0.1) { + issues.push(`High failure rate: ${(failureRate * 100).toFixed(1)}%`); + } + + // Check for queue backlog + if (metrics.waiting > 1000) { + issues.push(`Large queue backlog: ${metrics.waiting} jobs waiting`); + } + + // Check for slow processing + if (metrics.processingTime.avg > 30000) { + // 30 seconds + issues.push( + `Slow average processing time: ${(metrics.processingTime.avg / 1000).toFixed(1)}s` + ); + } + + // Check for stalled active jobs + if (metrics.active > 100) { + issues.push(`High number of active jobs: ${metrics.active}`); + } + + return { + isHealthy: issues.length === 0, + healthIssues: issues, + }; + } + + /** + * Get formatted metrics report + */ + async getReport(): Promise { + const metrics = await this.collect(); + + return ` +Queue Metrics Report +=================== +Status: ${metrics.isHealthy ? '✅ Healthy' : '⚠️ Issues Detected'} + +Job Counts: +- Waiting: ${metrics.waiting} +- Active: ${metrics.active} +- Completed: ${metrics.completed} +- Failed: ${metrics.failed} +- Delayed: ${metrics.delayed} +- Paused: ${metrics.paused} + +Performance: +- Avg Processing Time: ${(metrics.processingTime.avg / 1000).toFixed(2)}s +- Min/Max: ${(metrics.processingTime.min / 1000).toFixed(2)}s / ${(metrics.processingTime.max / 1000).toFixed(2)}s +- P95/P99: ${(metrics.processingTime.p95 / 1000).toFixed(2)}s / ${(metrics.processingTime.p99 / 1000).toFixed(2)}s + +Throughput: +- Completed/min: ${metrics.throughput.completedPerMinute} +- Failed/min: ${metrics.throughput.failedPerMinute} +- Total/min: ${metrics.throughput.totalPerMinute} + +${metrics.oldestWaitingJob ? `Oldest Waiting Job: ${metrics.oldestWaitingJob.toISOString()}` : 'No waiting jobs'} + +${metrics.healthIssues.length > 0 ? `\nHealth Issues:\n${metrics.healthIssues.map(issue => `- ${issue}`).join('\n')}` : ''} + `.trim(); + } + + /** + * Export metrics in Prometheus format + */ + async getPrometheusMetrics(): Promise { + const metrics = await this.collect(); + const queueName = this.queue.name; + + return ` +# HELP queue_jobs_total Total number of jobs by status +# TYPE queue_jobs_total gauge +queue_jobs_total{queue="${queueName}",status="waiting"} ${metrics.waiting} +queue_jobs_total{queue="${queueName}",status="active"} ${metrics.active} +queue_jobs_total{queue="${queueName}",status="completed"} ${metrics.completed} +queue_jobs_total{queue="${queueName}",status="failed"} ${metrics.failed} +queue_jobs_total{queue="${queueName}",status="delayed"} ${metrics.delayed} +queue_jobs_total{queue="${queueName}",status="paused"} ${metrics.paused} + +# HELP queue_processing_time_seconds Job processing time in seconds +# TYPE queue_processing_time_seconds summary +queue_processing_time_seconds{queue="${queueName}",quantile="0.5"} ${(metrics.processingTime.avg / 1000).toFixed(3)} +queue_processing_time_seconds{queue="${queueName}",quantile="0.95"} ${(metrics.processingTime.p95 / 1000).toFixed(3)} +queue_processing_time_seconds{queue="${queueName}",quantile="0.99"} ${(metrics.processingTime.p99 / 1000).toFixed(3)} +queue_processing_time_seconds_sum{queue="${queueName}"} ${((metrics.processingTime.avg * this.processingTimes.length) / 1000).toFixed(3)} +queue_processing_time_seconds_count{queue="${queueName}"} ${this.processingTimes.length} + +# HELP queue_throughput_per_minute Jobs processed per minute +# TYPE queue_throughput_per_minute gauge +queue_throughput_per_minute{queue="${queueName}",status="completed"} ${metrics.throughput.completedPerMinute} +queue_throughput_per_minute{queue="${queueName}",status="failed"} ${metrics.throughput.failedPerMinute} +queue_throughput_per_minute{queue="${queueName}",status="total"} ${metrics.throughput.totalPerMinute} + +# HELP queue_health Queue health status +# TYPE queue_health gauge +queue_health{queue="${queueName}"} ${metrics.isHealthy ? 1 : 0} + `.trim(); + } +} diff --git a/libs/services/queue/src/queue.ts b/libs/services/queue/src/queue.ts index e77b30b..2b1d884 100644 --- a/libs/services/queue/src/queue.ts +++ b/libs/services/queue/src/queue.ts @@ -1,372 +1,372 @@ -import { Queue as BullQueue, QueueEvents, Worker, type Job } from 'bullmq'; -import { getLogger } from '@stock-bot/logger'; -import { handlerRegistry } from '@stock-bot/types'; -import type { JobData, JobOptions, QueueStats, RedisConfig } from './types'; -import { getRedisConnection } from './utils'; - -const logger = getLogger('queue'); - -export interface QueueWorkerConfig { - workers?: number; - concurrency?: number; - startWorker?: boolean; -} - -/** - * Consolidated Queue class that handles both job operations and optional worker management - * Can be used as a simple job queue or with workers for automatic processing - */ -export class Queue { - private bullQueue: BullQueue; - private workers: Worker[] = []; - private queueEvents?: QueueEvents; - private queueName: string; - private redisConfig: RedisConfig; - - constructor( - queueName: string, - redisConfig: RedisConfig, - defaultJobOptions: JobOptions = {}, - config: QueueWorkerConfig = {} - ) { - this.queueName = queueName; - this.redisConfig = redisConfig; - - const connection = getRedisConnection(redisConfig); - - // Initialize BullMQ queue - this.bullQueue = new BullQueue(`{${queueName}}`, { - connection, - defaultJobOptions: { - removeOnComplete: 10, - removeOnFail: 5, - attempts: 3, - backoff: { - type: 'exponential', - delay: 1000, - }, - ...defaultJobOptions, - }, - }); - - // Initialize queue events if workers will be used - if (config.workers && config.workers > 0) { - this.queueEvents = new QueueEvents(`{${queueName}}`, { connection }); - } - - // Start workers if requested and not explicitly disabled - if (config.workers && config.workers > 0 && config.startWorker !== false) { - this.startWorkers(config.workers, config.concurrency || 1); - } - - logger.trace('Queue created', { - queueName, - workers: config.workers || 0, - concurrency: config.concurrency || 1, - }); - } - - /** - * Get the queue name - */ - getName(): string { - return this.queueName; - } - - /** - * Add a single job to the queue - */ - async add(name: string, data: JobData, options: JobOptions = {}): Promise { - logger.trace('Adding job', { queueName: this.queueName, jobName: name }); - return await this.bullQueue.add(name, data, options); - } - - /** - * Add multiple jobs to the queue in bulk - */ - async addBulk(jobs: Array<{ name: string; data: JobData; opts?: JobOptions }>): Promise { - logger.trace('Adding bulk jobs', { - queueName: this.queueName, - jobCount: jobs.length, - }); - return await this.bullQueue.addBulk(jobs); - } - - /** - * Add a scheduled job with cron-like pattern - */ - async addScheduledJob( - name: string, - data: JobData, - cronPattern: string, - options: JobOptions = {} - ): Promise { - const scheduledOptions: JobOptions = { - ...options, - repeat: { - pattern: cronPattern, - // Use job name as repeat key to prevent duplicates - key: `${this.queueName}:${name}`, - ...options.repeat, - }, - }; - - logger.info('Adding scheduled job', { - queueName: this.queueName, - jobName: name, - cronPattern, - repeatKey: scheduledOptions.repeat?.key, - immediately: scheduledOptions.repeat?.immediately, - }); - - return await this.bullQueue.add(name, data, scheduledOptions); - } - - /** - * Get queue statistics - */ - async getStats(): Promise { - const [waiting, active, completed, failed, delayed] = await Promise.all([ - this.bullQueue.getWaiting(), - this.bullQueue.getActive(), - this.bullQueue.getCompleted(), - this.bullQueue.getFailed(), - this.bullQueue.getDelayed(), - ]); - - const isPaused = await this.bullQueue.isPaused(); - - return { - waiting: waiting.length, - active: active.length, - completed: completed.length, - failed: failed.length, - delayed: delayed.length, - paused: isPaused, - workers: this.workers.length, - }; - } - - /** - * Get a specific job by ID - */ - async getJob(jobId: string): Promise { - return await this.bullQueue.getJob(jobId); - } - - /** - * Get jobs by state - */ - async getJobs( - states: Array<'waiting' | 'active' | 'completed' | 'failed' | 'delayed'>, - start = 0, - end = 100 - ): Promise { - return await this.bullQueue.getJobs(states, start, end); - } - - /** - * Pause the queue (stops processing new jobs) - */ - async pause(): Promise { - await this.bullQueue.pause(); - logger.info('Queue paused', { queueName: this.queueName }); - } - - /** - * Resume the queue - */ - async resume(): Promise { - await this.bullQueue.resume(); - logger.info('Queue resumed', { queueName: this.queueName }); - } - - /** - * Drain the queue (remove all jobs) - */ - async drain(delayed = false): Promise { - await this.bullQueue.drain(delayed); - logger.info('Queue drained', { queueName: this.queueName, delayed }); - } - - /** - * Clean completed and failed jobs - */ - async clean( - grace: number = 0, - limit: number = 100, - type: 'completed' | 'failed' = 'completed' - ): Promise { - await this.bullQueue.clean(grace, limit, type); - logger.debug('Queue cleaned', { queueName: this.queueName, type, grace, limit }); - } - - /** - * Wait until the queue is ready - */ - async waitUntilReady(): Promise { - await this.bullQueue.waitUntilReady(); - } - - /** - * Close the queue (cleanup resources) - */ - /** - * Close the queue (cleanup resources) - */ - async close(): Promise { - try { - // Close the queue itself - await this.bullQueue.close(); - logger.info('Queue closed', { queueName: this.queueName }); - - // Close queue events - if (this.queueEvents) { - await this.queueEvents.close(); - logger.debug('Queue events closed', { queueName: this.queueName }); - } - - // Close workers first - if (this.workers.length > 0) { - await Promise.all( - this.workers.map(async worker => { - return await worker.close(); - }) - ); - this.workers = []; - logger.debug('Workers closed', { queueName: this.queueName }); - } - } catch (error) { - logger.error('Error closing queue', { queueName: this.queueName, error }); - throw error; - } - } - - /** - * Start workers for this queue - */ - private startWorkers(workerCount: number, concurrency: number): void { - const connection = getRedisConnection(this.redisConfig); - - for (let i = 0; i < workerCount; i++) { - const worker = new Worker(`{${this.queueName}}`, this.processJob.bind(this), { - connection, - concurrency, - maxStalledCount: 3, - stalledInterval: 30000, - }); - - // Setup worker event handlers - worker.on('completed', job => { - logger.trace('Job completed', { - queueName: this.queueName, - jobId: job.id, - handler: job.data?.handler, - operation: job.data?.operation, - }); - }); - - worker.on('failed', (job, err) => { - logger.error('Job failed', { - queueName: this.queueName, - jobId: job?.id, - handler: job?.data?.handler, - operation: job?.data?.operation, - error: err.message, - }); - }); - - worker.on('error', error => { - logger.error('Worker error', { - queueName: this.queueName, - workerId: i, - error: error.message, - }); - }); - - this.workers.push(worker); - } - - logger.info('Workers started', { - queueName: this.queueName, - workerCount, - concurrency, - }); - } - - /** - * Process a job using the handler registry - */ - private async processJob(job: Job): Promise { - const { handler, operation, payload }: JobData = job.data; - - logger.trace('Processing job', { - id: job.id, - handler, - operation, - queueName: this.queueName, - }); - - try { - // Look up handler in registry - const jobHandler = handlerRegistry.getOperation(handler, operation); - - if (!jobHandler) { - throw new Error(`No handler found for ${handler}:${operation}`); - } - - const result = await jobHandler(payload); - - logger.trace('Job completed successfully', { - id: job.id, - handler, - operation, - queueName: this.queueName, - }); - - return result; - } catch (error) { - logger.error('Job processing failed', { - id: job.id, - handler, - operation, - queueName: this.queueName, - error: error instanceof Error ? error.message : String(error), - }); - throw error; - } - } - - /** - * Start workers manually (for delayed initialization) - */ - startWorkersManually(workerCount: number, concurrency: number = 1): void { - if (this.workers.length > 0) { - logger.warn('Workers already started for queue', { queueName: this.queueName }); - return; - } - - // Initialize queue events if not already done - if (!this.queueEvents) { - const connection = getRedisConnection(this.redisConfig); - this.queueEvents = new QueueEvents(`{${this.queueName}}`, { connection }); - } - - this.startWorkers(workerCount, concurrency); - } - - /** - * Get the number of active workers - */ - getWorkerCount(): number { - return this.workers.length; - } - - /** - * Get the underlying BullMQ queue (for advanced operations) - * @deprecated Use direct methods instead - */ - getBullQueue(): BullQueue { - return this.bullQueue; - } -} +import { Queue as BullQueue, QueueEvents, Worker, type Job } from 'bullmq'; +import { getLogger } from '@stock-bot/logger'; +import { handlerRegistry } from '@stock-bot/types'; +import type { JobData, JobOptions, QueueStats, RedisConfig } from './types'; +import { getRedisConnection } from './utils'; + +const logger = getLogger('queue'); + +export interface QueueWorkerConfig { + workers?: number; + concurrency?: number; + startWorker?: boolean; +} + +/** + * Consolidated Queue class that handles both job operations and optional worker management + * Can be used as a simple job queue or with workers for automatic processing + */ +export class Queue { + private bullQueue: BullQueue; + private workers: Worker[] = []; + private queueEvents?: QueueEvents; + private queueName: string; + private redisConfig: RedisConfig; + + constructor( + queueName: string, + redisConfig: RedisConfig, + defaultJobOptions: JobOptions = {}, + config: QueueWorkerConfig = {} + ) { + this.queueName = queueName; + this.redisConfig = redisConfig; + + const connection = getRedisConnection(redisConfig); + + // Initialize BullMQ queue + this.bullQueue = new BullQueue(`{${queueName}}`, { + connection, + defaultJobOptions: { + removeOnComplete: 10, + removeOnFail: 5, + attempts: 3, + backoff: { + type: 'exponential', + delay: 1000, + }, + ...defaultJobOptions, + }, + }); + + // Initialize queue events if workers will be used + if (config.workers && config.workers > 0) { + this.queueEvents = new QueueEvents(`{${queueName}}`, { connection }); + } + + // Start workers if requested and not explicitly disabled + if (config.workers && config.workers > 0 && config.startWorker !== false) { + this.startWorkers(config.workers, config.concurrency || 1); + } + + logger.trace('Queue created', { + queueName, + workers: config.workers || 0, + concurrency: config.concurrency || 1, + }); + } + + /** + * Get the queue name + */ + getName(): string { + return this.queueName; + } + + /** + * Add a single job to the queue + */ + async add(name: string, data: JobData, options: JobOptions = {}): Promise { + logger.trace('Adding job', { queueName: this.queueName, jobName: name }); + return await this.bullQueue.add(name, data, options); + } + + /** + * Add multiple jobs to the queue in bulk + */ + async addBulk(jobs: Array<{ name: string; data: JobData; opts?: JobOptions }>): Promise { + logger.trace('Adding bulk jobs', { + queueName: this.queueName, + jobCount: jobs.length, + }); + return await this.bullQueue.addBulk(jobs); + } + + /** + * Add a scheduled job with cron-like pattern + */ + async addScheduledJob( + name: string, + data: JobData, + cronPattern: string, + options: JobOptions = {} + ): Promise { + const scheduledOptions: JobOptions = { + ...options, + repeat: { + pattern: cronPattern, + // Use job name as repeat key to prevent duplicates + key: `${this.queueName}:${name}`, + ...options.repeat, + }, + }; + + logger.info('Adding scheduled job', { + queueName: this.queueName, + jobName: name, + cronPattern, + repeatKey: scheduledOptions.repeat?.key, + immediately: scheduledOptions.repeat?.immediately, + }); + + return await this.bullQueue.add(name, data, scheduledOptions); + } + + /** + * Get queue statistics + */ + async getStats(): Promise { + const [waiting, active, completed, failed, delayed] = await Promise.all([ + this.bullQueue.getWaiting(), + this.bullQueue.getActive(), + this.bullQueue.getCompleted(), + this.bullQueue.getFailed(), + this.bullQueue.getDelayed(), + ]); + + const isPaused = await this.bullQueue.isPaused(); + + return { + waiting: waiting.length, + active: active.length, + completed: completed.length, + failed: failed.length, + delayed: delayed.length, + paused: isPaused, + workers: this.workers.length, + }; + } + + /** + * Get a specific job by ID + */ + async getJob(jobId: string): Promise { + return await this.bullQueue.getJob(jobId); + } + + /** + * Get jobs by state + */ + async getJobs( + states: Array<'waiting' | 'active' | 'completed' | 'failed' | 'delayed'>, + start = 0, + end = 100 + ): Promise { + return await this.bullQueue.getJobs(states, start, end); + } + + /** + * Pause the queue (stops processing new jobs) + */ + async pause(): Promise { + await this.bullQueue.pause(); + logger.info('Queue paused', { queueName: this.queueName }); + } + + /** + * Resume the queue + */ + async resume(): Promise { + await this.bullQueue.resume(); + logger.info('Queue resumed', { queueName: this.queueName }); + } + + /** + * Drain the queue (remove all jobs) + */ + async drain(delayed = false): Promise { + await this.bullQueue.drain(delayed); + logger.info('Queue drained', { queueName: this.queueName, delayed }); + } + + /** + * Clean completed and failed jobs + */ + async clean( + grace: number = 0, + limit: number = 100, + type: 'completed' | 'failed' = 'completed' + ): Promise { + await this.bullQueue.clean(grace, limit, type); + logger.debug('Queue cleaned', { queueName: this.queueName, type, grace, limit }); + } + + /** + * Wait until the queue is ready + */ + async waitUntilReady(): Promise { + await this.bullQueue.waitUntilReady(); + } + + /** + * Close the queue (cleanup resources) + */ + /** + * Close the queue (cleanup resources) + */ + async close(): Promise { + try { + // Close the queue itself + await this.bullQueue.close(); + logger.info('Queue closed', { queueName: this.queueName }); + + // Close queue events + if (this.queueEvents) { + await this.queueEvents.close(); + logger.debug('Queue events closed', { queueName: this.queueName }); + } + + // Close workers first + if (this.workers.length > 0) { + await Promise.all( + this.workers.map(async worker => { + return await worker.close(); + }) + ); + this.workers = []; + logger.debug('Workers closed', { queueName: this.queueName }); + } + } catch (error) { + logger.error('Error closing queue', { queueName: this.queueName, error }); + throw error; + } + } + + /** + * Start workers for this queue + */ + private startWorkers(workerCount: number, concurrency: number): void { + const connection = getRedisConnection(this.redisConfig); + + for (let i = 0; i < workerCount; i++) { + const worker = new Worker(`{${this.queueName}}`, this.processJob.bind(this), { + connection, + concurrency, + maxStalledCount: 3, + stalledInterval: 30000, + }); + + // Setup worker event handlers + worker.on('completed', job => { + logger.trace('Job completed', { + queueName: this.queueName, + jobId: job.id, + handler: job.data?.handler, + operation: job.data?.operation, + }); + }); + + worker.on('failed', (job, err) => { + logger.error('Job failed', { + queueName: this.queueName, + jobId: job?.id, + handler: job?.data?.handler, + operation: job?.data?.operation, + error: err.message, + }); + }); + + worker.on('error', error => { + logger.error('Worker error', { + queueName: this.queueName, + workerId: i, + error: error.message, + }); + }); + + this.workers.push(worker); + } + + logger.info('Workers started', { + queueName: this.queueName, + workerCount, + concurrency, + }); + } + + /** + * Process a job using the handler registry + */ + private async processJob(job: Job): Promise { + const { handler, operation, payload }: JobData = job.data; + + logger.trace('Processing job', { + id: job.id, + handler, + operation, + queueName: this.queueName, + }); + + try { + // Look up handler in registry + const jobHandler = handlerRegistry.getOperation(handler, operation); + + if (!jobHandler) { + throw new Error(`No handler found for ${handler}:${operation}`); + } + + const result = await jobHandler(payload); + + logger.trace('Job completed successfully', { + id: job.id, + handler, + operation, + queueName: this.queueName, + }); + + return result; + } catch (error) { + logger.error('Job processing failed', { + id: job.id, + handler, + operation, + queueName: this.queueName, + error: error instanceof Error ? error.message : String(error), + }); + throw error; + } + } + + /** + * Start workers manually (for delayed initialization) + */ + startWorkersManually(workerCount: number, concurrency: number = 1): void { + if (this.workers.length > 0) { + logger.warn('Workers already started for queue', { queueName: this.queueName }); + return; + } + + // Initialize queue events if not already done + if (!this.queueEvents) { + const connection = getRedisConnection(this.redisConfig); + this.queueEvents = new QueueEvents(`{${this.queueName}}`, { connection }); + } + + this.startWorkers(workerCount, concurrency); + } + + /** + * Get the number of active workers + */ + getWorkerCount(): number { + return this.workers.length; + } + + /** + * Get the underlying BullMQ queue (for advanced operations) + * @deprecated Use direct methods instead + */ + getBullQueue(): BullQueue { + return this.bullQueue; + } +} diff --git a/libs/services/queue/src/rate-limiter.ts b/libs/services/queue/src/rate-limiter.ts index f8cf62a..ecb9d52 100644 --- a/libs/services/queue/src/rate-limiter.ts +++ b/libs/services/queue/src/rate-limiter.ts @@ -1,294 +1,327 @@ -import { RateLimiterRedis, RateLimiterRes } from 'rate-limiter-flexible'; -import { getLogger } from '@stock-bot/logger'; -import type { RateLimitConfig as BaseRateLimitConfig, RateLimitRule } from './types'; - -const logger = getLogger('rate-limiter'); - -// Extend the base config to add rate-limiter specific fields -export interface RateLimitConfig extends BaseRateLimitConfig { - keyPrefix?: string; -} - -export class QueueRateLimiter { - private limiters = new Map(); - private rules: RateLimitRule[] = []; - - constructor(private redisClient: ReturnType) {} - - /** - * Add a rate limit rule - */ - addRule(rule: RateLimitRule): void { - this.rules.push(rule); - - const key = this.getRuleKey(rule.level, rule.queueName, rule.handler, rule.operation); - const limiter = new RateLimiterRedis({ - storeClient: this.redisClient, - keyPrefix: `rl:${key}`, - points: rule.config.points, - duration: rule.config.duration, - blockDuration: rule.config.blockDuration || 0, - }); - - this.limiters.set(key, limiter); - - logger.info('Rate limit rule added', { - level: rule.level, - queueName: rule.queueName, - handler: rule.handler, - operation: rule.operation, - points: rule.config.points, - duration: rule.config.duration, - }); - } - - /** - * Check if a job can be processed based on rate limits - * Uses hierarchical precedence: operation > handler > queue > global - * The most specific matching rule takes precedence - */ - async checkLimit(queueName: string, handler: string, operation: string): Promise<{ - allowed: boolean; - retryAfter?: number; - remainingPoints?: number; - appliedRule?: RateLimitRule; - }> { - const applicableRule = this.getMostSpecificRule(queueName, handler, operation); - - if (!applicableRule) { - return { allowed: true }; - } - - const key = this.getRuleKey(applicableRule.level, applicableRule.queueName, applicableRule.handler, applicableRule.operation); - const limiter = this.limiters.get(key); - - if (!limiter) { - logger.warn('Rate limiter not found for rule', { key, rule: applicableRule }); - return { allowed: true }; - } - - try { - const result = await this.consumePoint(limiter, this.getConsumerKey(queueName, handler, operation)); - - return { - ...result, - appliedRule: applicableRule, - }; - } catch (error) { - logger.error('Rate limit check failed', { queueName, handler, operation, error }); - // On error, allow the request to proceed - return { allowed: true }; - } - } - - /** - * Get the most specific rule that applies to this job - * Precedence: operation > handler > queue > global - */ - private getMostSpecificRule(queueName: string, handler: string, operation: string): RateLimitRule | undefined { - // 1. Check for operation-specific rule (most specific) - let rule = this.rules.find(r => - r.level === 'operation' && - r.queueName === queueName && - r.handler === handler && - r.operation === operation - ); - if (rule) {return rule;} - - // 2. Check for handler-specific rule - rule = this.rules.find(r => - r.level === 'handler' && - r.queueName === queueName && - r.handler === handler - ); - if (rule) {return rule;} - - // 3. Check for queue-specific rule - rule = this.rules.find(r => - r.level === 'queue' && - r.queueName === queueName - ); - if (rule) {return rule;} - - // 4. Check for global rule (least specific) - rule = this.rules.find(r => r.level === 'global'); - return rule; - } - - /** - * Consume a point from the rate limiter - */ - private async consumePoint( - limiter: RateLimiterRedis, - key: string - ): Promise<{ allowed: boolean; retryAfter?: number; remainingPoints?: number }> { - try { - const result = await limiter.consume(key); - return { - allowed: true, - remainingPoints: result.remainingPoints, - }; - } catch (rejRes) { - if (rejRes instanceof RateLimiterRes) { - logger.warn('Rate limit exceeded', { - key, - retryAfter: rejRes.msBeforeNext, - }); - - return { - allowed: false, - retryAfter: rejRes.msBeforeNext, - remainingPoints: rejRes.remainingPoints, - }; - } - throw rejRes; - } - } - - /** - * Get rule key for storing rate limiter - */ - private getRuleKey(level: string, queueName?: string, handler?: string, operation?: string): string { - switch (level) { - case 'global': - return 'global'; - case 'queue': - return `queue:${queueName}`; - case 'handler': - return `handler:${queueName}:${handler}`; - case 'operation': - return `operation:${queueName}:${handler}:${operation}`; - default: - return level; - } - } - - /** - * Get consumer key for rate limiting (what gets counted) - */ - private getConsumerKey(queueName: string, handler: string, operation: string): string { - return `${queueName}:${handler}:${operation}`; - } - - /** - * Get current rate limit status for a queue/handler/operation - */ - async getStatus(queueName: string, handler: string, operation: string): Promise<{ - queueName: string; - handler: string; - operation: string; - appliedRule?: RateLimitRule; - limit?: { - level: string; - points: number; - duration: number; - remaining: number; - resetIn: number; - }; - }> { - const applicableRule = this.getMostSpecificRule(queueName, handler, operation); - - if (!applicableRule) { - return { - queueName, - handler, - operation, - }; - } - - const key = this.getRuleKey(applicableRule.level, applicableRule.queueName, applicableRule.handler, applicableRule.operation); - const limiter = this.limiters.get(key); - - if (!limiter) { - return { - queueName, - handler, - operation, - appliedRule: applicableRule, - }; - } - - try { - const consumerKey = this.getConsumerKey(queueName, handler, operation); - const result = await limiter.get(consumerKey); - - const limit = { - level: applicableRule.level, - points: limiter.points, - duration: limiter.duration, - remaining: result?.remainingPoints ?? limiter.points, - resetIn: result?.msBeforeNext ?? 0, - }; - - return { - queueName, - handler, - operation, - appliedRule: applicableRule, - limit, - }; - } catch (error) { - logger.error('Failed to get rate limit status', { queueName, handler, operation, error }); - return { - queueName, - handler, - operation, - appliedRule: applicableRule, - }; - } - } - - /** - * Reset rate limits for a specific consumer - */ - async reset(queueName: string, handler?: string, operation?: string): Promise { - if (handler && operation) { - // Reset specific operation - const consumerKey = this.getConsumerKey(queueName, handler, operation); - const rule = this.getMostSpecificRule(queueName, handler, operation); - - if (rule) { - const key = this.getRuleKey(rule.level, rule.queueName, rule.handler, rule.operation); - const limiter = this.limiters.get(key); - if (limiter) { - await limiter.delete(consumerKey); - } - } - } else { - // Reset broader scope - this is more complex with the new hierarchy - logger.warn('Broad reset not implemented yet', { queueName, handler, operation }); - } - - logger.info('Rate limits reset', { queueName, handler, operation }); - } - - /** - * Get all configured rate limit rules - */ - getRules(): RateLimitRule[] { - return [...this.rules]; - } - - /** - * Remove a rate limit rule - */ - removeRule(level: string, queueName?: string, handler?: string, operation?: string): boolean { - const key = this.getRuleKey(level, queueName, handler, operation); - const ruleIndex = this.rules.findIndex(r => - r.level === level && - (!queueName || r.queueName === queueName) && - (!handler || r.handler === handler) && - (!operation || r.operation === operation) - ); - - if (ruleIndex >= 0) { - this.rules.splice(ruleIndex, 1); - this.limiters.delete(key); - - logger.info('Rate limit rule removed', { level, queueName, handler, operation }); - return true; - } - - return false; - } -} \ No newline at end of file +import { RateLimiterRedis, RateLimiterRes } from 'rate-limiter-flexible'; +import { getLogger } from '@stock-bot/logger'; +import type { RateLimitConfig as BaseRateLimitConfig, RateLimitRule } from './types'; + +const logger = getLogger('rate-limiter'); + +// Extend the base config to add rate-limiter specific fields +export interface RateLimitConfig extends BaseRateLimitConfig { + keyPrefix?: string; +} + +export class QueueRateLimiter { + private limiters = new Map(); + private rules: RateLimitRule[] = []; + + constructor(private redisClient: ReturnType) {} + + /** + * Add a rate limit rule + */ + addRule(rule: RateLimitRule): void { + this.rules.push(rule); + + const key = this.getRuleKey(rule.level, rule.queueName, rule.handler, rule.operation); + const limiter = new RateLimiterRedis({ + storeClient: this.redisClient, + keyPrefix: `rl:${key}`, + points: rule.config.points, + duration: rule.config.duration, + blockDuration: rule.config.blockDuration || 0, + }); + + this.limiters.set(key, limiter); + + logger.info('Rate limit rule added', { + level: rule.level, + queueName: rule.queueName, + handler: rule.handler, + operation: rule.operation, + points: rule.config.points, + duration: rule.config.duration, + }); + } + + /** + * Check if a job can be processed based on rate limits + * Uses hierarchical precedence: operation > handler > queue > global + * The most specific matching rule takes precedence + */ + async checkLimit( + queueName: string, + handler: string, + operation: string + ): Promise<{ + allowed: boolean; + retryAfter?: number; + remainingPoints?: number; + appliedRule?: RateLimitRule; + }> { + const applicableRule = this.getMostSpecificRule(queueName, handler, operation); + + if (!applicableRule) { + return { allowed: true }; + } + + const key = this.getRuleKey( + applicableRule.level, + applicableRule.queueName, + applicableRule.handler, + applicableRule.operation + ); + const limiter = this.limiters.get(key); + + if (!limiter) { + logger.warn('Rate limiter not found for rule', { key, rule: applicableRule }); + return { allowed: true }; + } + + try { + const result = await this.consumePoint( + limiter, + this.getConsumerKey(queueName, handler, operation) + ); + + return { + ...result, + appliedRule: applicableRule, + }; + } catch (error) { + logger.error('Rate limit check failed', { queueName, handler, operation, error }); + // On error, allow the request to proceed + return { allowed: true }; + } + } + + /** + * Get the most specific rule that applies to this job + * Precedence: operation > handler > queue > global + */ + private getMostSpecificRule( + queueName: string, + handler: string, + operation: string + ): RateLimitRule | undefined { + // 1. Check for operation-specific rule (most specific) + let rule = this.rules.find( + r => + r.level === 'operation' && + r.queueName === queueName && + r.handler === handler && + r.operation === operation + ); + if (rule) { + return rule; + } + + // 2. Check for handler-specific rule + rule = this.rules.find( + r => r.level === 'handler' && r.queueName === queueName && r.handler === handler + ); + if (rule) { + return rule; + } + + // 3. Check for queue-specific rule + rule = this.rules.find(r => r.level === 'queue' && r.queueName === queueName); + if (rule) { + return rule; + } + + // 4. Check for global rule (least specific) + rule = this.rules.find(r => r.level === 'global'); + return rule; + } + + /** + * Consume a point from the rate limiter + */ + private async consumePoint( + limiter: RateLimiterRedis, + key: string + ): Promise<{ allowed: boolean; retryAfter?: number; remainingPoints?: number }> { + try { + const result = await limiter.consume(key); + return { + allowed: true, + remainingPoints: result.remainingPoints, + }; + } catch (rejRes) { + if (rejRes instanceof RateLimiterRes) { + logger.warn('Rate limit exceeded', { + key, + retryAfter: rejRes.msBeforeNext, + }); + + return { + allowed: false, + retryAfter: rejRes.msBeforeNext, + remainingPoints: rejRes.remainingPoints, + }; + } + throw rejRes; + } + } + + /** + * Get rule key for storing rate limiter + */ + private getRuleKey( + level: string, + queueName?: string, + handler?: string, + operation?: string + ): string { + switch (level) { + case 'global': + return 'global'; + case 'queue': + return `queue:${queueName}`; + case 'handler': + return `handler:${queueName}:${handler}`; + case 'operation': + return `operation:${queueName}:${handler}:${operation}`; + default: + return level; + } + } + + /** + * Get consumer key for rate limiting (what gets counted) + */ + private getConsumerKey(queueName: string, handler: string, operation: string): string { + return `${queueName}:${handler}:${operation}`; + } + + /** + * Get current rate limit status for a queue/handler/operation + */ + async getStatus( + queueName: string, + handler: string, + operation: string + ): Promise<{ + queueName: string; + handler: string; + operation: string; + appliedRule?: RateLimitRule; + limit?: { + level: string; + points: number; + duration: number; + remaining: number; + resetIn: number; + }; + }> { + const applicableRule = this.getMostSpecificRule(queueName, handler, operation); + + if (!applicableRule) { + return { + queueName, + handler, + operation, + }; + } + + const key = this.getRuleKey( + applicableRule.level, + applicableRule.queueName, + applicableRule.handler, + applicableRule.operation + ); + const limiter = this.limiters.get(key); + + if (!limiter) { + return { + queueName, + handler, + operation, + appliedRule: applicableRule, + }; + } + + try { + const consumerKey = this.getConsumerKey(queueName, handler, operation); + const result = await limiter.get(consumerKey); + + const limit = { + level: applicableRule.level, + points: limiter.points, + duration: limiter.duration, + remaining: result?.remainingPoints ?? limiter.points, + resetIn: result?.msBeforeNext ?? 0, + }; + + return { + queueName, + handler, + operation, + appliedRule: applicableRule, + limit, + }; + } catch (error) { + logger.error('Failed to get rate limit status', { queueName, handler, operation, error }); + return { + queueName, + handler, + operation, + appliedRule: applicableRule, + }; + } + } + + /** + * Reset rate limits for a specific consumer + */ + async reset(queueName: string, handler?: string, operation?: string): Promise { + if (handler && operation) { + // Reset specific operation + const consumerKey = this.getConsumerKey(queueName, handler, operation); + const rule = this.getMostSpecificRule(queueName, handler, operation); + + if (rule) { + const key = this.getRuleKey(rule.level, rule.queueName, rule.handler, rule.operation); + const limiter = this.limiters.get(key); + if (limiter) { + await limiter.delete(consumerKey); + } + } + } else { + // Reset broader scope - this is more complex with the new hierarchy + logger.warn('Broad reset not implemented yet', { queueName, handler, operation }); + } + + logger.info('Rate limits reset', { queueName, handler, operation }); + } + + /** + * Get all configured rate limit rules + */ + getRules(): RateLimitRule[] { + return [...this.rules]; + } + + /** + * Remove a rate limit rule + */ + removeRule(level: string, queueName?: string, handler?: string, operation?: string): boolean { + const key = this.getRuleKey(level, queueName, handler, operation); + const ruleIndex = this.rules.findIndex( + r => + r.level === level && + (!queueName || r.queueName === queueName) && + (!handler || r.handler === handler) && + (!operation || r.operation === operation) + ); + + if (ruleIndex >= 0) { + this.rules.splice(ruleIndex, 1); + this.limiters.delete(key); + + logger.info('Rate limit rule removed', { level, queueName, handler, operation }); + return true; + } + + return false; + } +} diff --git a/libs/services/queue/src/types.ts b/libs/services/queue/src/types.ts index 417091a..33d4dd9 100644 --- a/libs/services/queue/src/types.ts +++ b/libs/services/queue/src/types.ts @@ -71,7 +71,7 @@ export interface QueueOptions { enableMetrics?: boolean; enableDLQ?: boolean; enableRateLimit?: boolean; - rateLimitRules?: RateLimitRule[]; // Queue-specific rate limit rules + rateLimitRules?: RateLimitRule[]; // Queue-specific rate limit rules } export interface QueueManagerConfig { @@ -79,8 +79,8 @@ export interface QueueManagerConfig { defaultQueueOptions?: QueueOptions; enableScheduledJobs?: boolean; globalRateLimit?: RateLimitConfig; - rateLimitRules?: RateLimitRule[]; // Global rate limit rules - delayWorkerStart?: boolean; // If true, workers won't start automatically + rateLimitRules?: RateLimitRule[]; // Global rate limit rules + delayWorkerStart?: boolean; // If true, workers won't start automatically } export interface QueueStats { @@ -118,7 +118,7 @@ export interface BatchJobData { batchIndex: number; totalBatches: number; itemCount: number; - totalDelayHours: number; // Total time to distribute all batches + totalDelayHours: number; // Total time to distribute all batches } export interface HandlerInitializer { @@ -134,9 +134,9 @@ export interface RateLimitConfig { export interface RateLimitRule { level: 'global' | 'queue' | 'handler' | 'operation'; - queueName?: string; // For queue-level limits - handler?: string; // For handler-level limits - operation?: string; // For operation-level limits (most specific) + queueName?: string; // For queue-level limits + handler?: string; // For handler-level limits + operation?: string; // For operation-level limits (most specific) config: RateLimitConfig; } diff --git a/libs/services/queue/src/utils.ts b/libs/services/queue/src/utils.ts index 0c5e987..6c1d78b 100644 --- a/libs/services/queue/src/utils.ts +++ b/libs/services/queue/src/utils.ts @@ -5,7 +5,7 @@ import type { RedisConfig } from './types'; */ export function getRedisConnection(config: RedisConfig) { const isTest = process.env.NODE_ENV === 'test' || process.env['BUNIT'] === '1'; - + return { host: config.host, port: config.port, diff --git a/libs/services/queue/test/batch-processor.test.ts b/libs/services/queue/test/batch-processor.test.ts index 4c1f548..d98ad48 100644 --- a/libs/services/queue/test/batch-processor.test.ts +++ b/libs/services/queue/test/batch-processor.test.ts @@ -1,355 +1,364 @@ -import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; -import { QueueManager, Queue, handlerRegistry, processItems } from '../src'; - -// Suppress Redis connection errors in tests -process.on('unhandledRejection', (reason, promise) => { - if (reason && typeof reason === 'object' && 'message' in reason) { - const message = (reason as Error).message; - if (message.includes('Connection is closed') || - message.includes('Connection is in monitoring mode')) { - return; - } - } - console.error('Unhandled Rejection at:', promise, 'reason:', reason); -}); - -describe('Batch Processor', () => { - let queueManager: QueueManager; - let queue: Queue; - let queueName: string; - - const redisConfig = { - host: 'localhost', - port: 6379, - password: '', - db: 0, - }; - - - beforeEach(async () => { - // Clear handler registry - handlerRegistry.clear(); - - // Register test handler - handlerRegistry.register('batch-test', { - 'process-item': async (payload) => { - return { processed: true, data: payload }; - }, - 'generic': async (payload) => { - return { processed: true, data: payload }; - }, - 'process-batch-items': async (_batchData) => { - // This is called by the batch processor internally - return { batchProcessed: true }; - }, - }); - - // Use unique queue name per test to avoid conflicts - queueName = `batch-test-queue-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; - - // Reset and initialize singleton QueueManager for tests - await QueueManager.reset(); - queueManager = QueueManager.initialize({ - redis: redisConfig, - defaultQueueOptions: { - workers: 0, // No workers in tests - concurrency: 5, - }, - }); - - // Get queue using the new getQueue() method (batch cache is now auto-initialized) - queue = queueManager.getQueue(queueName); - // Note: Batch cache is now automatically initialized when getting the queue - - // Ensure completely clean state - wait for queue to be ready first - await queue.getBullQueue().waitUntilReady(); - - // Clear all job states - await queue.getBullQueue().drain(true); - await queue.getBullQueue().clean(0, 1000, 'completed'); - await queue.getBullQueue().clean(0, 1000, 'failed'); - await queue.getBullQueue().clean(0, 1000, 'active'); - await queue.getBullQueue().clean(0, 1000, 'waiting'); - await queue.getBullQueue().clean(0, 1000, 'delayed'); - - // Add a small delay to ensure cleanup is complete - await new Promise(resolve => setTimeout(resolve, 50)); - }); - - afterEach(async () => { - try { - // Clean up jobs first - if (queue) { - try { - await queue.getBullQueue().drain(true); - await queue.getBullQueue().clean(0, 1000, 'completed'); - await queue.getBullQueue().clean(0, 1000, 'failed'); - await queue.getBullQueue().clean(0, 1000, 'active'); - await queue.getBullQueue().clean(0, 1000, 'waiting'); - await queue.getBullQueue().clean(0, 1000, 'delayed'); - } catch { - // Ignore cleanup errors - } - await queue.close(); - } - - if (queueManager) { - await Promise.race([ - QueueManager.reset(), - new Promise((_, reject) => - setTimeout(() => reject(new Error('Shutdown timeout')), 3000) - ) - ]); - } - } catch (error) { - console.warn('Cleanup error:', error.message); - } finally { - handlerRegistry.clear(); - await new Promise(resolve => setTimeout(resolve, 100)); - } - }); - - describe('Direct Processing', () => { - test('should process items directly without batching', async () => { - const items = ['item1', 'item2', 'item3', 'item4', 'item5']; - - const result = await processItems(items, queueName, { - totalDelayHours: 0.001, // 3.6 seconds total - useBatching: false, - handler: 'batch-test', - operation: 'process-item', - priority: 1, - }); - - expect(result.mode).toBe('direct'); - expect(result.totalItems).toBe(5); - expect(result.jobsCreated).toBe(5); - - // Verify jobs were created - BullMQ has an issue where job ID "1" doesn't show up in state queries - // but exists when queried directly, so we need to check both ways - const [delayedJobs, waitingJobs, activeJobs, completedJobs, failedJobs, job1] = await Promise.all([ - queue.getBullQueue().getJobs(['delayed']), - queue.getBullQueue().getJobs(['waiting']), - queue.getBullQueue().getJobs(['active']), - queue.getBullQueue().getJobs(['completed']), - queue.getBullQueue().getJobs(['failed']), - queue.getBullQueue().getJob('1'), // Job 1 often doesn't show up in state queries - ]); - - const jobs = [...delayedJobs, ...waitingJobs, ...activeJobs, ...completedJobs, ...failedJobs]; - const ourJobs = jobs.filter(j => j.name === 'process-item' && j.data.handler === 'batch-test'); - - // Include job 1 if we found it directly but it wasn't in the state queries - if (job1 && job1.name === 'process-item' && job1.data.handler === 'batch-test' && !ourJobs.find(j => j.id === '1')) { - ourJobs.push(job1); - } - - expect(ourJobs.length).toBe(5); - - // Check delays are distributed - const delays = ourJobs.map(j => j.opts.delay || 0).sort((a, b) => a - b); - expect(delays[0]).toBe(0); - expect(delays[4]).toBeGreaterThan(delays[0]); - }); - - test('should process complex objects directly', async () => { - const items = [ - { id: 1, name: 'Product A', price: 100 }, - { id: 2, name: 'Product B', price: 200 }, - { id: 3, name: 'Product C', price: 300 }, - ]; - - const result = await processItems(items, queueName, { - totalDelayHours: 0.001, - useBatching: false, - handler: 'batch-test', - operation: 'process-item', - }); - - expect(result.jobsCreated).toBe(3); - - // Check job payloads - const jobs = await queue.getBullQueue().getJobs(['waiting', 'delayed']); - const ourJobs = jobs.filter(j => j.name === 'process-item' && j.data.handler === 'batch-test'); - const payloads = ourJobs.map(j => j.data.payload); - - expect(payloads).toContainEqual({ id: 1, name: 'Product A', price: 100 }); - expect(payloads).toContainEqual({ id: 2, name: 'Product B', price: 200 }); - expect(payloads).toContainEqual({ id: 3, name: 'Product C', price: 300 }); - }); - }); - - describe('Batch Processing', () => { - test('should process items in batches', async () => { - const items = Array.from({ length: 50 }, (_, i) => ({ id: i, value: `item-${i}` })); - - const result = await processItems(items, queueName, { - totalDelayHours: 0.001, - useBatching: true, - batchSize: 10, - handler: 'batch-test', - operation: 'process-item', - }); - - expect(result.mode).toBe('batch'); - expect(result.totalItems).toBe(50); - expect(result.batchesCreated).toBe(5); // 50 items / 10 per batch - expect(result.jobsCreated).toBe(5); // 5 batch jobs - - // Verify batch jobs were created - const jobs = await queue.getBullQueue().getJobs(['delayed', 'waiting']); - const batchJobs = jobs.filter(j => j.name === 'process-batch'); - expect(batchJobs.length).toBe(5); - }); - - test('should handle different batch sizes', async () => { - const items = Array.from({ length: 23 }, (_, i) => i); - - const result = await processItems(items, queueName, { - totalDelayHours: 0.001, - useBatching: true, - batchSize: 7, - handler: 'batch-test', - operation: 'process-item', - }); - - expect(result.batchesCreated).toBe(4); // 23/7 = 3.28, rounded up to 4 - expect(result.jobsCreated).toBe(4); - }); - - test('should store batch payloads in cache', async () => { - const items = [ - { type: 'A', data: 'test1' }, - { type: 'B', data: 'test2' }, - ]; - - const result = await processItems(items, queueName, { - totalDelayHours: 0.001, - useBatching: true, - batchSize: 2, - handler: 'batch-test', - operation: 'process-item', - ttl: 3600, // 1 hour TTL - }); - - expect(result.jobsCreated).toBe(1); - - // Get the batch job - const jobs = await queue.getBullQueue().getJobs(['waiting', 'delayed']); - expect(jobs.length).toBe(1); - - const batchJob = jobs[0]; - expect(batchJob.data.payload.payloadKey).toBeDefined(); - expect(batchJob.data.payload.itemCount).toBe(2); - }); - }); - - describe('Empty and Edge Cases', () => { - test('should handle empty item list', async () => { - const result = await processItems([], queueName, { - totalDelayHours: 1, - handler: 'batch-test', - operation: 'process-item', - }); - - expect(result.totalItems).toBe(0); - expect(result.jobsCreated).toBe(0); - expect(result.duration).toBeDefined(); - }); - - test('should handle single item', async () => { - const result = await processItems(['single-item'], queueName, { - totalDelayHours: 0.001, - handler: 'batch-test', - operation: 'process-item', - }); - - expect(result.totalItems).toBe(1); - expect(result.jobsCreated).toBe(1); - }); - - test('should handle large batch with delays', async () => { - const items = Array.from({ length: 100 }, (_, i) => ({ index: i })); - - const result = await processItems(items, queueName, { - totalDelayHours: 0.01, // 36 seconds total - useBatching: true, - batchSize: 25, - handler: 'batch-test', - operation: 'process-item', - }); - - expect(result.batchesCreated).toBe(4); // 100/25 - expect(result.jobsCreated).toBe(4); - - // Check delays are distributed - const jobs = await queue.getBullQueue().getJobs(['delayed', 'waiting']); - const delays = jobs.map(j => j.opts.delay || 0).sort((a, b) => a - b); - - expect(delays[0]).toBe(0); // First batch has no delay - expect(delays[3]).toBeGreaterThan(0); // Last batch has delay - }); - }); - - describe('Job Options', () => { - test('should respect custom job options', async () => { - const items = ['a', 'b', 'c']; - - await processItems(items, queueName, { - totalDelayHours: 0, - handler: 'batch-test', - operation: 'process-item', - priority: 5, - retries: 10, - removeOnComplete: 100, - removeOnFail: 50, - }); - - // Check all states including job ID "1" specifically (as it often doesn't show up in state queries) - const [waitingJobs, delayedJobs, job1, job2, job3] = await Promise.all([ - queue.getBullQueue().getJobs(['waiting']), - queue.getBullQueue().getJobs(['delayed']), - queue.getBullQueue().getJob('1'), - queue.getBullQueue().getJob('2'), - queue.getBullQueue().getJob('3'), - ]); - - const jobs = [...waitingJobs, ...delayedJobs]; - // Add any missing jobs that exist but don't show up in state queries - [job1, job2, job3].forEach(job => { - if (job && !jobs.find(j => j.id === job.id)) { - jobs.push(job); - } - }); - - expect(jobs.length).toBe(3); - - jobs.forEach(job => { - expect(job.opts.priority).toBe(5); - expect(job.opts.attempts).toBe(10); - expect(job.opts.removeOnComplete).toBe(100); - expect(job.opts.removeOnFail).toBe(50); - }); - }); - - test('should set handler and operation correctly', async () => { - // Register custom handler for this test - handlerRegistry.register('custom-handler', { - 'custom-operation': async (payload) => { - return { processed: true, data: payload }; - }, - }); - - await processItems(['test'], queueName, { - totalDelayHours: 0, - handler: 'custom-handler', - operation: 'custom-operation', - }); - - const jobs = await queue.getBullQueue().getJobs(['waiting']); - expect(jobs.length).toBe(1); - expect(jobs[0].data.handler).toBe('custom-handler'); - expect(jobs[0].data.operation).toBe('custom-operation'); - }); - }); -}); \ No newline at end of file +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { handlerRegistry, processItems, Queue, QueueManager } from '../src'; + +// Suppress Redis connection errors in tests +process.on('unhandledRejection', (reason, promise) => { + if (reason && typeof reason === 'object' && 'message' in reason) { + const message = (reason as Error).message; + if ( + message.includes('Connection is closed') || + message.includes('Connection is in monitoring mode') + ) { + return; + } + } + console.error('Unhandled Rejection at:', promise, 'reason:', reason); +}); + +describe('Batch Processor', () => { + let queueManager: QueueManager; + let queue: Queue; + let queueName: string; + + const redisConfig = { + host: 'localhost', + port: 6379, + password: '', + db: 0, + }; + + beforeEach(async () => { + // Clear handler registry + handlerRegistry.clear(); + + // Register test handler + handlerRegistry.register('batch-test', { + 'process-item': async payload => { + return { processed: true, data: payload }; + }, + generic: async payload => { + return { processed: true, data: payload }; + }, + 'process-batch-items': async _batchData => { + // This is called by the batch processor internally + return { batchProcessed: true }; + }, + }); + + // Use unique queue name per test to avoid conflicts + queueName = `batch-test-queue-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + + // Reset and initialize singleton QueueManager for tests + await QueueManager.reset(); + queueManager = QueueManager.initialize({ + redis: redisConfig, + defaultQueueOptions: { + workers: 0, // No workers in tests + concurrency: 5, + }, + }); + + // Get queue using the new getQueue() method (batch cache is now auto-initialized) + queue = queueManager.getQueue(queueName); + // Note: Batch cache is now automatically initialized when getting the queue + + // Ensure completely clean state - wait for queue to be ready first + await queue.getBullQueue().waitUntilReady(); + + // Clear all job states + await queue.getBullQueue().drain(true); + await queue.getBullQueue().clean(0, 1000, 'completed'); + await queue.getBullQueue().clean(0, 1000, 'failed'); + await queue.getBullQueue().clean(0, 1000, 'active'); + await queue.getBullQueue().clean(0, 1000, 'waiting'); + await queue.getBullQueue().clean(0, 1000, 'delayed'); + + // Add a small delay to ensure cleanup is complete + await new Promise(resolve => setTimeout(resolve, 50)); + }); + + afterEach(async () => { + try { + // Clean up jobs first + if (queue) { + try { + await queue.getBullQueue().drain(true); + await queue.getBullQueue().clean(0, 1000, 'completed'); + await queue.getBullQueue().clean(0, 1000, 'failed'); + await queue.getBullQueue().clean(0, 1000, 'active'); + await queue.getBullQueue().clean(0, 1000, 'waiting'); + await queue.getBullQueue().clean(0, 1000, 'delayed'); + } catch { + // Ignore cleanup errors + } + await queue.close(); + } + + if (queueManager) { + await Promise.race([ + QueueManager.reset(), + new Promise((_, reject) => setTimeout(() => reject(new Error('Shutdown timeout')), 3000)), + ]); + } + } catch (error) { + console.warn('Cleanup error:', error.message); + } finally { + handlerRegistry.clear(); + await new Promise(resolve => setTimeout(resolve, 100)); + } + }); + + describe('Direct Processing', () => { + test('should process items directly without batching', async () => { + const items = ['item1', 'item2', 'item3', 'item4', 'item5']; + + const result = await processItems(items, queueName, { + totalDelayHours: 0.001, // 3.6 seconds total + useBatching: false, + handler: 'batch-test', + operation: 'process-item', + priority: 1, + }); + + expect(result.mode).toBe('direct'); + expect(result.totalItems).toBe(5); + expect(result.jobsCreated).toBe(5); + + // Verify jobs were created - BullMQ has an issue where job ID "1" doesn't show up in state queries + // but exists when queried directly, so we need to check both ways + const [delayedJobs, waitingJobs, activeJobs, completedJobs, failedJobs, job1] = + await Promise.all([ + queue.getBullQueue().getJobs(['delayed']), + queue.getBullQueue().getJobs(['waiting']), + queue.getBullQueue().getJobs(['active']), + queue.getBullQueue().getJobs(['completed']), + queue.getBullQueue().getJobs(['failed']), + queue.getBullQueue().getJob('1'), // Job 1 often doesn't show up in state queries + ]); + + const jobs = [...delayedJobs, ...waitingJobs, ...activeJobs, ...completedJobs, ...failedJobs]; + const ourJobs = jobs.filter( + j => j.name === 'process-item' && j.data.handler === 'batch-test' + ); + + // Include job 1 if we found it directly but it wasn't in the state queries + if ( + job1 && + job1.name === 'process-item' && + job1.data.handler === 'batch-test' && + !ourJobs.find(j => j.id === '1') + ) { + ourJobs.push(job1); + } + + expect(ourJobs.length).toBe(5); + + // Check delays are distributed + const delays = ourJobs.map(j => j.opts.delay || 0).sort((a, b) => a - b); + expect(delays[0]).toBe(0); + expect(delays[4]).toBeGreaterThan(delays[0]); + }); + + test('should process complex objects directly', async () => { + const items = [ + { id: 1, name: 'Product A', price: 100 }, + { id: 2, name: 'Product B', price: 200 }, + { id: 3, name: 'Product C', price: 300 }, + ]; + + const result = await processItems(items, queueName, { + totalDelayHours: 0.001, + useBatching: false, + handler: 'batch-test', + operation: 'process-item', + }); + + expect(result.jobsCreated).toBe(3); + + // Check job payloads + const jobs = await queue.getBullQueue().getJobs(['waiting', 'delayed']); + const ourJobs = jobs.filter( + j => j.name === 'process-item' && j.data.handler === 'batch-test' + ); + const payloads = ourJobs.map(j => j.data.payload); + + expect(payloads).toContainEqual({ id: 1, name: 'Product A', price: 100 }); + expect(payloads).toContainEqual({ id: 2, name: 'Product B', price: 200 }); + expect(payloads).toContainEqual({ id: 3, name: 'Product C', price: 300 }); + }); + }); + + describe('Batch Processing', () => { + test('should process items in batches', async () => { + const items = Array.from({ length: 50 }, (_, i) => ({ id: i, value: `item-${i}` })); + + const result = await processItems(items, queueName, { + totalDelayHours: 0.001, + useBatching: true, + batchSize: 10, + handler: 'batch-test', + operation: 'process-item', + }); + + expect(result.mode).toBe('batch'); + expect(result.totalItems).toBe(50); + expect(result.batchesCreated).toBe(5); // 50 items / 10 per batch + expect(result.jobsCreated).toBe(5); // 5 batch jobs + + // Verify batch jobs were created + const jobs = await queue.getBullQueue().getJobs(['delayed', 'waiting']); + const batchJobs = jobs.filter(j => j.name === 'process-batch'); + expect(batchJobs.length).toBe(5); + }); + + test('should handle different batch sizes', async () => { + const items = Array.from({ length: 23 }, (_, i) => i); + + const result = await processItems(items, queueName, { + totalDelayHours: 0.001, + useBatching: true, + batchSize: 7, + handler: 'batch-test', + operation: 'process-item', + }); + + expect(result.batchesCreated).toBe(4); // 23/7 = 3.28, rounded up to 4 + expect(result.jobsCreated).toBe(4); + }); + + test('should store batch payloads in cache', async () => { + const items = [ + { type: 'A', data: 'test1' }, + { type: 'B', data: 'test2' }, + ]; + + const result = await processItems(items, queueName, { + totalDelayHours: 0.001, + useBatching: true, + batchSize: 2, + handler: 'batch-test', + operation: 'process-item', + ttl: 3600, // 1 hour TTL + }); + + expect(result.jobsCreated).toBe(1); + + // Get the batch job + const jobs = await queue.getBullQueue().getJobs(['waiting', 'delayed']); + expect(jobs.length).toBe(1); + + const batchJob = jobs[0]; + expect(batchJob.data.payload.payloadKey).toBeDefined(); + expect(batchJob.data.payload.itemCount).toBe(2); + }); + }); + + describe('Empty and Edge Cases', () => { + test('should handle empty item list', async () => { + const result = await processItems([], queueName, { + totalDelayHours: 1, + handler: 'batch-test', + operation: 'process-item', + }); + + expect(result.totalItems).toBe(0); + expect(result.jobsCreated).toBe(0); + expect(result.duration).toBeDefined(); + }); + + test('should handle single item', async () => { + const result = await processItems(['single-item'], queueName, { + totalDelayHours: 0.001, + handler: 'batch-test', + operation: 'process-item', + }); + + expect(result.totalItems).toBe(1); + expect(result.jobsCreated).toBe(1); + }); + + test('should handle large batch with delays', async () => { + const items = Array.from({ length: 100 }, (_, i) => ({ index: i })); + + const result = await processItems(items, queueName, { + totalDelayHours: 0.01, // 36 seconds total + useBatching: true, + batchSize: 25, + handler: 'batch-test', + operation: 'process-item', + }); + + expect(result.batchesCreated).toBe(4); // 100/25 + expect(result.jobsCreated).toBe(4); + + // Check delays are distributed + const jobs = await queue.getBullQueue().getJobs(['delayed', 'waiting']); + const delays = jobs.map(j => j.opts.delay || 0).sort((a, b) => a - b); + + expect(delays[0]).toBe(0); // First batch has no delay + expect(delays[3]).toBeGreaterThan(0); // Last batch has delay + }); + }); + + describe('Job Options', () => { + test('should respect custom job options', async () => { + const items = ['a', 'b', 'c']; + + await processItems(items, queueName, { + totalDelayHours: 0, + handler: 'batch-test', + operation: 'process-item', + priority: 5, + retries: 10, + removeOnComplete: 100, + removeOnFail: 50, + }); + + // Check all states including job ID "1" specifically (as it often doesn't show up in state queries) + const [waitingJobs, delayedJobs, job1, job2, job3] = await Promise.all([ + queue.getBullQueue().getJobs(['waiting']), + queue.getBullQueue().getJobs(['delayed']), + queue.getBullQueue().getJob('1'), + queue.getBullQueue().getJob('2'), + queue.getBullQueue().getJob('3'), + ]); + + const jobs = [...waitingJobs, ...delayedJobs]; + // Add any missing jobs that exist but don't show up in state queries + [job1, job2, job3].forEach(job => { + if (job && !jobs.find(j => j.id === job.id)) { + jobs.push(job); + } + }); + + expect(jobs.length).toBe(3); + + jobs.forEach(job => { + expect(job.opts.priority).toBe(5); + expect(job.opts.attempts).toBe(10); + expect(job.opts.removeOnComplete).toBe(100); + expect(job.opts.removeOnFail).toBe(50); + }); + }); + + test('should set handler and operation correctly', async () => { + // Register custom handler for this test + handlerRegistry.register('custom-handler', { + 'custom-operation': async payload => { + return { processed: true, data: payload }; + }, + }); + + await processItems(['test'], queueName, { + totalDelayHours: 0, + handler: 'custom-handler', + operation: 'custom-operation', + }); + + const jobs = await queue.getBullQueue().getJobs(['waiting']); + expect(jobs.length).toBe(1); + expect(jobs[0].data.handler).toBe('custom-handler'); + expect(jobs[0].data.operation).toBe('custom-operation'); + }); + }); +}); diff --git a/libs/services/queue/test/dlq-handler.test.ts b/libs/services/queue/test/dlq-handler.test.ts index 7b7a335..657404a 100644 --- a/libs/services/queue/test/dlq-handler.test.ts +++ b/libs/services/queue/test/dlq-handler.test.ts @@ -1,357 +1,379 @@ -import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; -import { Queue, Worker } from 'bullmq'; -import { DeadLetterQueueHandler } from '../src/dlq-handler'; -import { getRedisConnection } from '../src/utils'; - -// Suppress Redis connection errors in tests -process.on('unhandledRejection', (reason, promise) => { - if (reason && typeof reason === 'object' && 'message' in reason) { - const message = (reason as Error).message; - if (message.includes('Connection is closed') || - message.includes('Connection is in monitoring mode')) { - return; - } - } - console.error('Unhandled Rejection at:', promise, 'reason:', reason); -}); - -describe('DeadLetterQueueHandler', () => { - let mainQueue: Queue; - let dlqHandler: DeadLetterQueueHandler; - let worker: Worker; - let connection: any; - - const redisConfig = { - host: 'localhost', - port: 6379, - password: '', - db: 0, - }; - - beforeEach(async () => { - connection = getRedisConnection(redisConfig); - - // Create main queue - mainQueue = new Queue('test-queue', { connection }); - - // Create DLQ handler - dlqHandler = new DeadLetterQueueHandler(mainQueue, connection, { - maxRetries: 3, - retryDelay: 100, - alertThreshold: 5, - cleanupAge: 24, - }); - }); - - afterEach(async () => { - try { - if (worker) { - await worker.close(); - } - await dlqHandler.shutdown(); - await mainQueue.close(); - } catch { - // Ignore cleanup errors - } - await new Promise(resolve => setTimeout(resolve, 50)); - }); - - describe('Failed Job Handling', () => { - test('should move job to DLQ after max retries', async () => { - let attemptCount = 0; - - // Create worker that always fails - worker = new Worker('test-queue', async () => { - attemptCount++; - throw new Error('Job failed'); - }, { - connection, - autorun: false, - }); - - // Add job with limited attempts - const _job = await mainQueue.add('failing-job', { test: true }, { - attempts: 3, - backoff: { type: 'fixed', delay: 50 }, - }); - - // Process job manually - await worker.run(); - - // Wait for retries - await new Promise(resolve => setTimeout(resolve, 300)); - - // Job should have failed 3 times - expect(attemptCount).toBe(3); - - // Check if job was moved to DLQ - const dlqStats = await dlqHandler.getStats(); - expect(dlqStats.total).toBe(1); - expect(dlqStats.byJobName['failing-job']).toBe(1); - }); - - test('should track failure count correctly', async () => { - const job = await mainQueue.add('test-job', { data: 'test' }); - const error = new Error('Test error'); - - // Simulate multiple failures - await dlqHandler.handleFailedJob(job, error); - await dlqHandler.handleFailedJob(job, error); - - // On third failure with max attempts reached, should move to DLQ - job.attemptsMade = 3; - job.opts.attempts = 3; - await dlqHandler.handleFailedJob(job, error); - - const stats = await dlqHandler.getStats(); - expect(stats.total).toBe(1); - }); - }); - - describe('DLQ Statistics', () => { - test('should provide detailed statistics', async () => { - // Add some failed jobs to DLQ - const dlq = new Queue(`test-queue-dlq`, { connection }); - - await dlq.add('failed-job', { - originalJob: { - id: '1', - name: 'job-type-a', - data: { test: true }, - attemptsMade: 3, - }, - error: { message: 'Error 1' }, - movedToDLQAt: new Date().toISOString(), - }); - - await dlq.add('failed-job', { - originalJob: { - id: '2', - name: 'job-type-b', - data: { test: true }, - attemptsMade: 3, - }, - error: { message: 'Error 2' }, - movedToDLQAt: new Date().toISOString(), - }); - - const stats = await dlqHandler.getStats(); - expect(stats.total).toBe(2); - expect(stats.recent).toBe(2); // Both are recent - expect(Object.keys(stats.byJobName).length).toBe(2); - expect(stats.oldestJob).toBeDefined(); - - await dlq.close(); - }); - - test('should count recent jobs correctly', async () => { - const dlq = new Queue(`test-queue-dlq`, { connection }); - - // Add old job (25 hours ago) - const oldTimestamp = Date.now() - 25 * 60 * 60 * 1000; - await dlq.add('failed-job', { - originalJob: { id: '1', name: 'old-job' }, - error: { message: 'Old error' }, - movedToDLQAt: new Date(oldTimestamp).toISOString(), - }, { timestamp: oldTimestamp }); - - // Add recent job - await dlq.add('failed-job', { - originalJob: { id: '2', name: 'recent-job' }, - error: { message: 'Recent error' }, - movedToDLQAt: new Date().toISOString(), - }); - - const stats = await dlqHandler.getStats(); - expect(stats.total).toBe(2); - expect(stats.recent).toBe(1); // Only one is recent - - await dlq.close(); - }); - }); - - describe('DLQ Retry', () => { - test('should retry jobs from DLQ', async () => { - const dlq = new Queue(`test-queue-dlq`, { connection }); - - // Add failed jobs to DLQ - await dlq.add('failed-job', { - originalJob: { - id: '1', - name: 'retry-job', - data: { retry: true }, - opts: { priority: 1 }, - }, - error: { message: 'Failed' }, - movedToDLQAt: new Date().toISOString(), - }); - - await dlq.add('failed-job', { - originalJob: { - id: '2', - name: 'retry-job-2', - data: { retry: true }, - opts: {}, - }, - error: { message: 'Failed' }, - movedToDLQAt: new Date().toISOString(), - }); - - // Retry jobs - const retriedCount = await dlqHandler.retryDLQJobs(10); - expect(retriedCount).toBe(2); - - // Check main queue has the retried jobs - const mainQueueJobs = await mainQueue.getWaiting(); - expect(mainQueueJobs.length).toBe(2); - expect(mainQueueJobs[0].name).toBe('retry-job'); - expect(mainQueueJobs[0].data).toEqual({ retry: true }); - - // DLQ should be empty - const dlqJobs = await dlq.getCompleted(); - expect(dlqJobs.length).toBe(0); - - await dlq.close(); - }); - - test('should respect retry limit', async () => { - const dlq = new Queue(`test-queue-dlq`, { connection }); - - // Add 5 failed jobs - for (let i = 0; i < 5; i++) { - await dlq.add('failed-job', { - originalJob: { - id: `${i}`, - name: `job-${i}`, - data: { index: i }, - }, - error: { message: 'Failed' }, - movedToDLQAt: new Date().toISOString(), - }); - } - - // Retry only 3 jobs - const retriedCount = await dlqHandler.retryDLQJobs(3); - expect(retriedCount).toBe(3); - - // Check counts - const mainQueueJobs = await mainQueue.getWaiting(); - expect(mainQueueJobs.length).toBe(3); - - const remainingDLQ = await dlq.getCompleted(); - expect(remainingDLQ.length).toBe(2); - - await dlq.close(); - }); - }); - - describe('DLQ Cleanup', () => { - test('should cleanup old DLQ entries', async () => { - const dlq = new Queue(`test-queue-dlq`, { connection }); - - // Add old job (25 hours ago) - const oldTimestamp = Date.now() - 25 * 60 * 60 * 1000; - await dlq.add('failed-job', { - originalJob: { id: '1', name: 'old-job' }, - error: { message: 'Old error' }, - }, { timestamp: oldTimestamp }); - - // Add recent job (1 hour ago) - const recentTimestamp = Date.now() - 1 * 60 * 60 * 1000; - await dlq.add('failed-job', { - originalJob: { id: '2', name: 'recent-job' }, - error: { message: 'Recent error' }, - }, { timestamp: recentTimestamp }); - - // Run cleanup (24 hour threshold) - const removedCount = await dlqHandler.cleanup(); - expect(removedCount).toBe(1); - - // Check remaining jobs - const remaining = await dlq.getCompleted(); - expect(remaining.length).toBe(1); - expect(remaining[0].data.originalJob.name).toBe('recent-job'); - - await dlq.close(); - }); - }); - - describe('Failed Job Inspection', () => { - test('should inspect failed jobs', async () => { - const dlq = new Queue(`test-queue-dlq`, { connection }); - - // Add failed jobs with different error types - await dlq.add('failed-job', { - originalJob: { - id: '1', - name: 'network-job', - data: { url: 'https://api.example.com' }, - attemptsMade: 3, - }, - error: { - message: 'Network timeout', - stack: 'Error: Network timeout\n at ...', - name: 'NetworkError', - }, - movedToDLQAt: '2024-01-01T10:00:00Z', - }); - - await dlq.add('failed-job', { - originalJob: { - id: '2', - name: 'parse-job', - data: { input: 'invalid-json' }, - attemptsMade: 2, - }, - error: { - message: 'Invalid JSON', - stack: 'SyntaxError: Invalid JSON\n at ...', - name: 'SyntaxError', - }, - movedToDLQAt: '2024-01-01T11:00:00Z', - }); - - const failedJobs = await dlqHandler.inspectFailedJobs(10); - expect(failedJobs.length).toBe(2); - - expect(failedJobs[0]).toMatchObject({ - id: '1', - name: 'network-job', - data: { url: 'https://api.example.com' }, - error: { - message: 'Network timeout', - name: 'NetworkError', - }, - failedAt: '2024-01-01T10:00:00Z', - attempts: 3, - }); - - await dlq.close(); - }); - }); - - describe('Alert Threshold', () => { - test('should detect when alert threshold is exceeded', async () => { - const dlq = new Queue(`test-queue-dlq`, { connection }); - - // Add jobs to exceed threshold (5) - for (let i = 0; i < 6; i++) { - await dlq.add('failed-job', { - originalJob: { - id: `${i}`, - name: `job-${i}`, - data: { index: i }, - }, - error: { message: 'Failed' }, - movedToDLQAt: new Date().toISOString(), - }); - } - - const stats = await dlqHandler.getStats(); - expect(stats.total).toBe(6); - // In a real implementation, this would trigger alerts - - await dlq.close(); - }); - }); -}); \ No newline at end of file +import { Queue, Worker } from 'bullmq'; +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { DeadLetterQueueHandler } from '../src/dlq-handler'; +import { getRedisConnection } from '../src/utils'; + +// Suppress Redis connection errors in tests +process.on('unhandledRejection', (reason, promise) => { + if (reason && typeof reason === 'object' && 'message' in reason) { + const message = (reason as Error).message; + if ( + message.includes('Connection is closed') || + message.includes('Connection is in monitoring mode') + ) { + return; + } + } + console.error('Unhandled Rejection at:', promise, 'reason:', reason); +}); + +describe('DeadLetterQueueHandler', () => { + let mainQueue: Queue; + let dlqHandler: DeadLetterQueueHandler; + let worker: Worker; + let connection: any; + + const redisConfig = { + host: 'localhost', + port: 6379, + password: '', + db: 0, + }; + + beforeEach(async () => { + connection = getRedisConnection(redisConfig); + + // Create main queue + mainQueue = new Queue('test-queue', { connection }); + + // Create DLQ handler + dlqHandler = new DeadLetterQueueHandler(mainQueue, connection, { + maxRetries: 3, + retryDelay: 100, + alertThreshold: 5, + cleanupAge: 24, + }); + }); + + afterEach(async () => { + try { + if (worker) { + await worker.close(); + } + await dlqHandler.shutdown(); + await mainQueue.close(); + } catch { + // Ignore cleanup errors + } + await new Promise(resolve => setTimeout(resolve, 50)); + }); + + describe('Failed Job Handling', () => { + test('should move job to DLQ after max retries', async () => { + let attemptCount = 0; + + // Create worker that always fails + worker = new Worker( + 'test-queue', + async () => { + attemptCount++; + throw new Error('Job failed'); + }, + { + connection, + autorun: false, + } + ); + + // Add job with limited attempts + const _job = await mainQueue.add( + 'failing-job', + { test: true }, + { + attempts: 3, + backoff: { type: 'fixed', delay: 50 }, + } + ); + + // Process job manually + await worker.run(); + + // Wait for retries + await new Promise(resolve => setTimeout(resolve, 300)); + + // Job should have failed 3 times + expect(attemptCount).toBe(3); + + // Check if job was moved to DLQ + const dlqStats = await dlqHandler.getStats(); + expect(dlqStats.total).toBe(1); + expect(dlqStats.byJobName['failing-job']).toBe(1); + }); + + test('should track failure count correctly', async () => { + const job = await mainQueue.add('test-job', { data: 'test' }); + const error = new Error('Test error'); + + // Simulate multiple failures + await dlqHandler.handleFailedJob(job, error); + await dlqHandler.handleFailedJob(job, error); + + // On third failure with max attempts reached, should move to DLQ + job.attemptsMade = 3; + job.opts.attempts = 3; + await dlqHandler.handleFailedJob(job, error); + + const stats = await dlqHandler.getStats(); + expect(stats.total).toBe(1); + }); + }); + + describe('DLQ Statistics', () => { + test('should provide detailed statistics', async () => { + // Add some failed jobs to DLQ + const dlq = new Queue(`test-queue-dlq`, { connection }); + + await dlq.add('failed-job', { + originalJob: { + id: '1', + name: 'job-type-a', + data: { test: true }, + attemptsMade: 3, + }, + error: { message: 'Error 1' }, + movedToDLQAt: new Date().toISOString(), + }); + + await dlq.add('failed-job', { + originalJob: { + id: '2', + name: 'job-type-b', + data: { test: true }, + attemptsMade: 3, + }, + error: { message: 'Error 2' }, + movedToDLQAt: new Date().toISOString(), + }); + + const stats = await dlqHandler.getStats(); + expect(stats.total).toBe(2); + expect(stats.recent).toBe(2); // Both are recent + expect(Object.keys(stats.byJobName).length).toBe(2); + expect(stats.oldestJob).toBeDefined(); + + await dlq.close(); + }); + + test('should count recent jobs correctly', async () => { + const dlq = new Queue(`test-queue-dlq`, { connection }); + + // Add old job (25 hours ago) + const oldTimestamp = Date.now() - 25 * 60 * 60 * 1000; + await dlq.add( + 'failed-job', + { + originalJob: { id: '1', name: 'old-job' }, + error: { message: 'Old error' }, + movedToDLQAt: new Date(oldTimestamp).toISOString(), + }, + { timestamp: oldTimestamp } + ); + + // Add recent job + await dlq.add('failed-job', { + originalJob: { id: '2', name: 'recent-job' }, + error: { message: 'Recent error' }, + movedToDLQAt: new Date().toISOString(), + }); + + const stats = await dlqHandler.getStats(); + expect(stats.total).toBe(2); + expect(stats.recent).toBe(1); // Only one is recent + + await dlq.close(); + }); + }); + + describe('DLQ Retry', () => { + test('should retry jobs from DLQ', async () => { + const dlq = new Queue(`test-queue-dlq`, { connection }); + + // Add failed jobs to DLQ + await dlq.add('failed-job', { + originalJob: { + id: '1', + name: 'retry-job', + data: { retry: true }, + opts: { priority: 1 }, + }, + error: { message: 'Failed' }, + movedToDLQAt: new Date().toISOString(), + }); + + await dlq.add('failed-job', { + originalJob: { + id: '2', + name: 'retry-job-2', + data: { retry: true }, + opts: {}, + }, + error: { message: 'Failed' }, + movedToDLQAt: new Date().toISOString(), + }); + + // Retry jobs + const retriedCount = await dlqHandler.retryDLQJobs(10); + expect(retriedCount).toBe(2); + + // Check main queue has the retried jobs + const mainQueueJobs = await mainQueue.getWaiting(); + expect(mainQueueJobs.length).toBe(2); + expect(mainQueueJobs[0].name).toBe('retry-job'); + expect(mainQueueJobs[0].data).toEqual({ retry: true }); + + // DLQ should be empty + const dlqJobs = await dlq.getCompleted(); + expect(dlqJobs.length).toBe(0); + + await dlq.close(); + }); + + test('should respect retry limit', async () => { + const dlq = new Queue(`test-queue-dlq`, { connection }); + + // Add 5 failed jobs + for (let i = 0; i < 5; i++) { + await dlq.add('failed-job', { + originalJob: { + id: `${i}`, + name: `job-${i}`, + data: { index: i }, + }, + error: { message: 'Failed' }, + movedToDLQAt: new Date().toISOString(), + }); + } + + // Retry only 3 jobs + const retriedCount = await dlqHandler.retryDLQJobs(3); + expect(retriedCount).toBe(3); + + // Check counts + const mainQueueJobs = await mainQueue.getWaiting(); + expect(mainQueueJobs.length).toBe(3); + + const remainingDLQ = await dlq.getCompleted(); + expect(remainingDLQ.length).toBe(2); + + await dlq.close(); + }); + }); + + describe('DLQ Cleanup', () => { + test('should cleanup old DLQ entries', async () => { + const dlq = new Queue(`test-queue-dlq`, { connection }); + + // Add old job (25 hours ago) + const oldTimestamp = Date.now() - 25 * 60 * 60 * 1000; + await dlq.add( + 'failed-job', + { + originalJob: { id: '1', name: 'old-job' }, + error: { message: 'Old error' }, + }, + { timestamp: oldTimestamp } + ); + + // Add recent job (1 hour ago) + const recentTimestamp = Date.now() - 1 * 60 * 60 * 1000; + await dlq.add( + 'failed-job', + { + originalJob: { id: '2', name: 'recent-job' }, + error: { message: 'Recent error' }, + }, + { timestamp: recentTimestamp } + ); + + // Run cleanup (24 hour threshold) + const removedCount = await dlqHandler.cleanup(); + expect(removedCount).toBe(1); + + // Check remaining jobs + const remaining = await dlq.getCompleted(); + expect(remaining.length).toBe(1); + expect(remaining[0].data.originalJob.name).toBe('recent-job'); + + await dlq.close(); + }); + }); + + describe('Failed Job Inspection', () => { + test('should inspect failed jobs', async () => { + const dlq = new Queue(`test-queue-dlq`, { connection }); + + // Add failed jobs with different error types + await dlq.add('failed-job', { + originalJob: { + id: '1', + name: 'network-job', + data: { url: 'https://api.example.com' }, + attemptsMade: 3, + }, + error: { + message: 'Network timeout', + stack: 'Error: Network timeout\n at ...', + name: 'NetworkError', + }, + movedToDLQAt: '2024-01-01T10:00:00Z', + }); + + await dlq.add('failed-job', { + originalJob: { + id: '2', + name: 'parse-job', + data: { input: 'invalid-json' }, + attemptsMade: 2, + }, + error: { + message: 'Invalid JSON', + stack: 'SyntaxError: Invalid JSON\n at ...', + name: 'SyntaxError', + }, + movedToDLQAt: '2024-01-01T11:00:00Z', + }); + + const failedJobs = await dlqHandler.inspectFailedJobs(10); + expect(failedJobs.length).toBe(2); + + expect(failedJobs[0]).toMatchObject({ + id: '1', + name: 'network-job', + data: { url: 'https://api.example.com' }, + error: { + message: 'Network timeout', + name: 'NetworkError', + }, + failedAt: '2024-01-01T10:00:00Z', + attempts: 3, + }); + + await dlq.close(); + }); + }); + + describe('Alert Threshold', () => { + test('should detect when alert threshold is exceeded', async () => { + const dlq = new Queue(`test-queue-dlq`, { connection }); + + // Add jobs to exceed threshold (5) + for (let i = 0; i < 6; i++) { + await dlq.add('failed-job', { + originalJob: { + id: `${i}`, + name: `job-${i}`, + data: { index: i }, + }, + error: { message: 'Failed' }, + movedToDLQAt: new Date().toISOString(), + }); + } + + const stats = await dlqHandler.getStats(); + expect(stats.total).toBe(6); + // In a real implementation, this would trigger alerts + + await dlq.close(); + }); + }); +}); diff --git a/libs/services/queue/test/queue-integration.test.ts b/libs/services/queue/test/queue-integration.test.ts index 4bf1f63..3f633c8 100644 --- a/libs/services/queue/test/queue-integration.test.ts +++ b/libs/services/queue/test/queue-integration.test.ts @@ -1,12 +1,14 @@ -import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; -import { QueueManager, handlerRegistry } from '../src'; +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { handlerRegistry, QueueManager } from '../src'; // Suppress Redis connection errors in tests process.on('unhandledRejection', (reason, promise) => { if (reason && typeof reason === 'object' && 'message' in reason) { const message = (reason as Error).message; - if (message.includes('Connection is closed') || - message.includes('Connection is in monitoring mode')) { + if ( + message.includes('Connection is closed') || + message.includes('Connection is in monitoring mode') + ) { // Suppress these specific Redis errors in tests return; } @@ -34,9 +36,7 @@ describe('QueueManager Integration Tests', () => { try { await Promise.race([ queueManager.shutdown(), - new Promise((_, reject) => - setTimeout(() => reject(new Error('Shutdown timeout')), 3000) - ) + new Promise((_, reject) => setTimeout(() => reject(new Error('Shutdown timeout')), 3000)), ]); } catch (error) { // Ignore shutdown errors in tests @@ -45,10 +45,10 @@ describe('QueueManager Integration Tests', () => { queueManager = null as any; } } - + // Clear handler registry to prevent conflicts handlerRegistry.clear(); - + // Add delay to allow connections to close await new Promise(resolve => setTimeout(resolve, 100)); }); diff --git a/libs/services/queue/test/queue-manager.test.ts b/libs/services/queue/test/queue-manager.test.ts index becfb00..7f33be4 100644 --- a/libs/services/queue/test/queue-manager.test.ts +++ b/libs/services/queue/test/queue-manager.test.ts @@ -1,371 +1,371 @@ -import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; -import { handlerRegistry, QueueManager } from '../src'; - -// Suppress Redis connection errors in tests -process.on('unhandledRejection', (reason, promise) => { - if (reason && typeof reason === 'object' && 'message' in reason) { - const message = (reason as Error).message; - if (message.includes('Connection is closed') || - message.includes('Connection is in monitoring mode')) { - return; - } - } - console.error('Unhandled Rejection at:', promise, 'reason:', reason); -}); - -describe('QueueManager', () => { - let queueManager: QueueManager; - - // Use local Redis/Dragonfly - const redisConfig = { - host: 'localhost', - port: 6379, - password: '', - db: 0, - }; - - beforeEach(() => { - handlerRegistry.clear(); - }); - - afterEach(async () => { - if (queueManager) { - try { - await Promise.race([ - queueManager.shutdown(), - new Promise((_, reject) => - setTimeout(() => reject(new Error('Shutdown timeout')), 3000) - ) - ]); - } catch (error) { - console.warn('Shutdown error:', error.message); - } finally { - queueManager = null as any; - } - } - - handlerRegistry.clear(); - await new Promise(resolve => setTimeout(resolve, 100)); - }); - - describe('Basic Operations', () => { - test('should initialize queue manager', async () => { - queueManager = new QueueManager({ - queueName: 'test-queue', - redis: redisConfig, - workers: 1, - concurrency: 5, - }); - - await queueManager.initialize(); - expect(queueManager.queueName).toBe('test-queue'); - }); - - test('should add and process a job', async () => { - let processedPayload: any; - - // Register handler - handlerRegistry.register('test-handler', { - 'test-operation': async payload => { - processedPayload = payload; - return { success: true, data: payload }; - }, - }); - - queueManager = new QueueManager({ - queueName: 'test-queue', - redis: redisConfig, - workers: 1, - }); - - await queueManager.initialize(); - - // Add job - const job = await queueManager.add('test-job', { - handler: 'test-handler', - operation: 'test-operation', - payload: { message: 'Hello, Queue!' }, - }); - - expect(job.name).toBe('test-job'); - - // Wait for processing - await new Promise(resolve => setTimeout(resolve, 100)); - - expect(processedPayload).toEqual({ message: 'Hello, Queue!' }); - }); - - test('should handle missing handler gracefully', async () => { - queueManager = new QueueManager({ - queueName: 'test-queue', - redis: redisConfig, - workers: 1, - }); - - await queueManager.initialize(); - - const job = await queueManager.add('test-job', { - handler: 'non-existent', - operation: 'test-operation', - payload: { test: true }, - }); - - // Wait for job to fail - await new Promise(resolve => setTimeout(resolve, 100)); - - const failed = await job.isFailed(); - expect(failed).toBe(true); - }); - - test('should add multiple jobs in bulk', async () => { - let processedCount = 0; - - handlerRegistry.register('bulk-handler', { - process: async _payload => { - processedCount++; - return { processed: true }; - }, - }); - - queueManager = new QueueManager({ - queueName: 'test-queue', - redis: redisConfig, - workers: 2, - concurrency: 5, - }); - - await queueManager.initialize(); - - const jobs = await queueManager.addBulk([ - { - name: 'job1', - data: { handler: 'bulk-handler', operation: 'process', payload: { id: 1 } }, - }, - { - name: 'job2', - data: { handler: 'bulk-handler', operation: 'process', payload: { id: 2 } }, - }, - { - name: 'job3', - data: { handler: 'bulk-handler', operation: 'process', payload: { id: 3 } }, - }, - ]); - - expect(jobs.length).toBe(3); - - // Wait for processing - await new Promise(resolve => setTimeout(resolve, 200)); - - expect(processedCount).toBe(3); - }); - - test('should get queue statistics', async () => { - queueManager = new QueueManager({ - queueName: 'test-queue', - redis: redisConfig, - workers: 0, // No workers, jobs will stay in waiting - }); - - await queueManager.initialize(); - - // Add some jobs - await queueManager.add('job1', { - handler: 'test', - operation: 'test', - payload: { id: 1 }, - }); - - await queueManager.add('job2', { - handler: 'test', - operation: 'test', - payload: { id: 2 }, - }); - - const stats = await queueManager.getStats(); - - expect(stats.waiting).toBe(2); - expect(stats.active).toBe(0); - expect(stats.completed).toBe(0); - expect(stats.failed).toBe(0); - }); - - test('should pause and resume queue', async () => { - let processedCount = 0; - - handlerRegistry.register('pause-test', { - process: async () => { - processedCount++; - return { ok: true }; - }, - }); - - queueManager = new QueueManager({ - queueName: 'test-queue', - redis: redisConfig, - workers: 1, - }); - - await queueManager.initialize(); - - // Pause queue - await queueManager.pause(); - - // Add job while paused - await queueManager.add('job1', { - handler: 'pause-test', - operation: 'process', - payload: {}, - }); - - // Wait a bit - job should not be processed - await new Promise(resolve => setTimeout(resolve, 100)); - expect(processedCount).toBe(0); - - // Resume queue - await queueManager.resume(); - - // Wait for processing - await new Promise(resolve => setTimeout(resolve, 100)); - expect(processedCount).toBe(1); - }); - }); - - describe('Scheduled Jobs', () => { - test('should register and process scheduled jobs', async () => { - let executionCount = 0; - - handlerRegistry.registerWithSchedule({ - name: 'scheduled-handler', - operations: { - 'scheduled-task': async _payload => { - executionCount++; - return { executed: true, timestamp: Date.now() }; - }, - }, - scheduledJobs: [ - { - type: 'test-schedule', - operation: 'scheduled-task', - payload: { test: true }, - cronPattern: '*/1 * * * * *', // Every second - description: 'Test scheduled job', - }, - ], - }); - - queueManager = new QueueManager({ - queueName: 'test-queue', - redis: redisConfig, - workers: 1, - enableScheduledJobs: true, - }); - - await queueManager.initialize(); - - // Wait for scheduled job to execute - await new Promise(resolve => setTimeout(resolve, 2500)); - - expect(executionCount).toBeGreaterThanOrEqual(2); - }); - }); - - describe('Error Handling', () => { - test('should handle job errors with retries', async () => { - let attemptCount = 0; - - handlerRegistry.register('retry-handler', { - 'failing-operation': async () => { - attemptCount++; - if (attemptCount < 3) { - throw new Error(`Attempt ${attemptCount} failed`); - } - return { success: true }; - }, - }); - - queueManager = new QueueManager({ - queueName: 'test-queue', - redis: redisConfig, - workers: 1, - defaultJobOptions: { - attempts: 3, - backoff: { - type: 'fixed', - delay: 50, - }, - }, - }); - - await queueManager.initialize(); - - const job = await queueManager.add('retry-job', { - handler: 'retry-handler', - operation: 'failing-operation', - payload: {}, - }); - - // Wait for retries - await new Promise(resolve => setTimeout(resolve, 500)); - - const completed = await job.isCompleted(); - expect(completed).toBe(true); - expect(attemptCount).toBe(3); - }); - }); - - describe('Multiple Handlers', () => { - test('should handle multiple handlers with different operations', async () => { - const results: any[] = []; - - handlerRegistry.register('handler-a', { - 'operation-1': async payload => { - results.push({ handler: 'a', op: '1', payload }); - return { handler: 'a', op: '1' }; - }, - 'operation-2': async payload => { - results.push({ handler: 'a', op: '2', payload }); - return { handler: 'a', op: '2' }; - }, - }); - - handlerRegistry.register('handler-b', { - 'operation-1': async payload => { - results.push({ handler: 'b', op: '1', payload }); - return { handler: 'b', op: '1' }; - }, - }); - - queueManager = new QueueManager({ - queueName: 'test-queue', - redis: redisConfig, - workers: 2, - }); - - await queueManager.initialize(); - - // Add jobs for different handlers - await queueManager.addBulk([ - { - name: 'job1', - data: { handler: 'handler-a', operation: 'operation-1', payload: { id: 1 } }, - }, - { - name: 'job2', - data: { handler: 'handler-a', operation: 'operation-2', payload: { id: 2 } }, - }, - { - name: 'job3', - data: { handler: 'handler-b', operation: 'operation-1', payload: { id: 3 } }, - }, - ]); - - // Wait for processing - await new Promise(resolve => setTimeout(resolve, 200)); - - expect(results.length).toBe(3); - expect(results).toContainEqual({ handler: 'a', op: '1', payload: { id: 1 } }); - expect(results).toContainEqual({ handler: 'a', op: '2', payload: { id: 2 } }); - expect(results).toContainEqual({ handler: 'b', op: '1', payload: { id: 3 } }); - }); - }); -}); +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { handlerRegistry, QueueManager } from '../src'; + +// Suppress Redis connection errors in tests +process.on('unhandledRejection', (reason, promise) => { + if (reason && typeof reason === 'object' && 'message' in reason) { + const message = (reason as Error).message; + if ( + message.includes('Connection is closed') || + message.includes('Connection is in monitoring mode') + ) { + return; + } + } + console.error('Unhandled Rejection at:', promise, 'reason:', reason); +}); + +describe('QueueManager', () => { + let queueManager: QueueManager; + + // Use local Redis/Dragonfly + const redisConfig = { + host: 'localhost', + port: 6379, + password: '', + db: 0, + }; + + beforeEach(() => { + handlerRegistry.clear(); + }); + + afterEach(async () => { + if (queueManager) { + try { + await Promise.race([ + queueManager.shutdown(), + new Promise((_, reject) => setTimeout(() => reject(new Error('Shutdown timeout')), 3000)), + ]); + } catch (error) { + console.warn('Shutdown error:', error.message); + } finally { + queueManager = null as any; + } + } + + handlerRegistry.clear(); + await new Promise(resolve => setTimeout(resolve, 100)); + }); + + describe('Basic Operations', () => { + test('should initialize queue manager', async () => { + queueManager = new QueueManager({ + queueName: 'test-queue', + redis: redisConfig, + workers: 1, + concurrency: 5, + }); + + await queueManager.initialize(); + expect(queueManager.queueName).toBe('test-queue'); + }); + + test('should add and process a job', async () => { + let processedPayload: any; + + // Register handler + handlerRegistry.register('test-handler', { + 'test-operation': async payload => { + processedPayload = payload; + return { success: true, data: payload }; + }, + }); + + queueManager = new QueueManager({ + queueName: 'test-queue', + redis: redisConfig, + workers: 1, + }); + + await queueManager.initialize(); + + // Add job + const job = await queueManager.add('test-job', { + handler: 'test-handler', + operation: 'test-operation', + payload: { message: 'Hello, Queue!' }, + }); + + expect(job.name).toBe('test-job'); + + // Wait for processing + await new Promise(resolve => setTimeout(resolve, 100)); + + expect(processedPayload).toEqual({ message: 'Hello, Queue!' }); + }); + + test('should handle missing handler gracefully', async () => { + queueManager = new QueueManager({ + queueName: 'test-queue', + redis: redisConfig, + workers: 1, + }); + + await queueManager.initialize(); + + const job = await queueManager.add('test-job', { + handler: 'non-existent', + operation: 'test-operation', + payload: { test: true }, + }); + + // Wait for job to fail + await new Promise(resolve => setTimeout(resolve, 100)); + + const failed = await job.isFailed(); + expect(failed).toBe(true); + }); + + test('should add multiple jobs in bulk', async () => { + let processedCount = 0; + + handlerRegistry.register('bulk-handler', { + process: async _payload => { + processedCount++; + return { processed: true }; + }, + }); + + queueManager = new QueueManager({ + queueName: 'test-queue', + redis: redisConfig, + workers: 2, + concurrency: 5, + }); + + await queueManager.initialize(); + + const jobs = await queueManager.addBulk([ + { + name: 'job1', + data: { handler: 'bulk-handler', operation: 'process', payload: { id: 1 } }, + }, + { + name: 'job2', + data: { handler: 'bulk-handler', operation: 'process', payload: { id: 2 } }, + }, + { + name: 'job3', + data: { handler: 'bulk-handler', operation: 'process', payload: { id: 3 } }, + }, + ]); + + expect(jobs.length).toBe(3); + + // Wait for processing + await new Promise(resolve => setTimeout(resolve, 200)); + + expect(processedCount).toBe(3); + }); + + test('should get queue statistics', async () => { + queueManager = new QueueManager({ + queueName: 'test-queue', + redis: redisConfig, + workers: 0, // No workers, jobs will stay in waiting + }); + + await queueManager.initialize(); + + // Add some jobs + await queueManager.add('job1', { + handler: 'test', + operation: 'test', + payload: { id: 1 }, + }); + + await queueManager.add('job2', { + handler: 'test', + operation: 'test', + payload: { id: 2 }, + }); + + const stats = await queueManager.getStats(); + + expect(stats.waiting).toBe(2); + expect(stats.active).toBe(0); + expect(stats.completed).toBe(0); + expect(stats.failed).toBe(0); + }); + + test('should pause and resume queue', async () => { + let processedCount = 0; + + handlerRegistry.register('pause-test', { + process: async () => { + processedCount++; + return { ok: true }; + }, + }); + + queueManager = new QueueManager({ + queueName: 'test-queue', + redis: redisConfig, + workers: 1, + }); + + await queueManager.initialize(); + + // Pause queue + await queueManager.pause(); + + // Add job while paused + await queueManager.add('job1', { + handler: 'pause-test', + operation: 'process', + payload: {}, + }); + + // Wait a bit - job should not be processed + await new Promise(resolve => setTimeout(resolve, 100)); + expect(processedCount).toBe(0); + + // Resume queue + await queueManager.resume(); + + // Wait for processing + await new Promise(resolve => setTimeout(resolve, 100)); + expect(processedCount).toBe(1); + }); + }); + + describe('Scheduled Jobs', () => { + test('should register and process scheduled jobs', async () => { + let executionCount = 0; + + handlerRegistry.registerWithSchedule({ + name: 'scheduled-handler', + operations: { + 'scheduled-task': async _payload => { + executionCount++; + return { executed: true, timestamp: Date.now() }; + }, + }, + scheduledJobs: [ + { + type: 'test-schedule', + operation: 'scheduled-task', + payload: { test: true }, + cronPattern: '*/1 * * * * *', // Every second + description: 'Test scheduled job', + }, + ], + }); + + queueManager = new QueueManager({ + queueName: 'test-queue', + redis: redisConfig, + workers: 1, + enableScheduledJobs: true, + }); + + await queueManager.initialize(); + + // Wait for scheduled job to execute + await new Promise(resolve => setTimeout(resolve, 2500)); + + expect(executionCount).toBeGreaterThanOrEqual(2); + }); + }); + + describe('Error Handling', () => { + test('should handle job errors with retries', async () => { + let attemptCount = 0; + + handlerRegistry.register('retry-handler', { + 'failing-operation': async () => { + attemptCount++; + if (attemptCount < 3) { + throw new Error(`Attempt ${attemptCount} failed`); + } + return { success: true }; + }, + }); + + queueManager = new QueueManager({ + queueName: 'test-queue', + redis: redisConfig, + workers: 1, + defaultJobOptions: { + attempts: 3, + backoff: { + type: 'fixed', + delay: 50, + }, + }, + }); + + await queueManager.initialize(); + + const job = await queueManager.add('retry-job', { + handler: 'retry-handler', + operation: 'failing-operation', + payload: {}, + }); + + // Wait for retries + await new Promise(resolve => setTimeout(resolve, 500)); + + const completed = await job.isCompleted(); + expect(completed).toBe(true); + expect(attemptCount).toBe(3); + }); + }); + + describe('Multiple Handlers', () => { + test('should handle multiple handlers with different operations', async () => { + const results: any[] = []; + + handlerRegistry.register('handler-a', { + 'operation-1': async payload => { + results.push({ handler: 'a', op: '1', payload }); + return { handler: 'a', op: '1' }; + }, + 'operation-2': async payload => { + results.push({ handler: 'a', op: '2', payload }); + return { handler: 'a', op: '2' }; + }, + }); + + handlerRegistry.register('handler-b', { + 'operation-1': async payload => { + results.push({ handler: 'b', op: '1', payload }); + return { handler: 'b', op: '1' }; + }, + }); + + queueManager = new QueueManager({ + queueName: 'test-queue', + redis: redisConfig, + workers: 2, + }); + + await queueManager.initialize(); + + // Add jobs for different handlers + await queueManager.addBulk([ + { + name: 'job1', + data: { handler: 'handler-a', operation: 'operation-1', payload: { id: 1 } }, + }, + { + name: 'job2', + data: { handler: 'handler-a', operation: 'operation-2', payload: { id: 2 } }, + }, + { + name: 'job3', + data: { handler: 'handler-b', operation: 'operation-1', payload: { id: 3 } }, + }, + ]); + + // Wait for processing + await new Promise(resolve => setTimeout(resolve, 200)); + + expect(results.length).toBe(3); + expect(results).toContainEqual({ handler: 'a', op: '1', payload: { id: 1 } }); + expect(results).toContainEqual({ handler: 'a', op: '2', payload: { id: 2 } }); + expect(results).toContainEqual({ handler: 'b', op: '1', payload: { id: 3 } }); + }); + }); +}); diff --git a/libs/services/queue/test/queue-metrics.test.ts b/libs/services/queue/test/queue-metrics.test.ts index 4c8acb5..d6fd985 100644 --- a/libs/services/queue/test/queue-metrics.test.ts +++ b/libs/services/queue/test/queue-metrics.test.ts @@ -1,303 +1,327 @@ -import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; -import { Queue, QueueEvents, Worker } from 'bullmq'; -import { QueueMetricsCollector } from '../src/queue-metrics'; -import { getRedisConnection } from '../src/utils'; - -// Suppress Redis connection errors in tests -process.on('unhandledRejection', (reason, promise) => { - if (reason && typeof reason === 'object' && 'message' in reason) { - const message = (reason as Error).message; - if (message.includes('Connection is closed') || - message.includes('Connection is in monitoring mode')) { - return; - } - } - console.error('Unhandled Rejection at:', promise, 'reason:', reason); -}); - -describe('QueueMetricsCollector', () => { - let queue: Queue; - let queueEvents: QueueEvents; - let metricsCollector: QueueMetricsCollector; - let worker: Worker; - let connection: any; - - const redisConfig = { - host: 'localhost', - port: 6379, - password: '', - db: 0, - }; - - beforeEach(async () => { - connection = getRedisConnection(redisConfig); - - // Create queue and events - queue = new Queue('metrics-test-queue', { connection }); - queueEvents = new QueueEvents('metrics-test-queue', { connection }); - - // Create metrics collector - metricsCollector = new QueueMetricsCollector(queue, queueEvents); - - // Wait for connections - await queue.waitUntilReady(); - await queueEvents.waitUntilReady(); - }); - - afterEach(async () => { - try { - if (worker) { - await worker.close(); - } - await queueEvents.close(); - await queue.close(); - } catch { - // Ignore cleanup errors - } - await new Promise(resolve => setTimeout(resolve, 50)); - }); - - describe('Job Count Metrics', () => { - test('should collect basic job counts', async () => { - // Add jobs in different states - await queue.add('waiting-job', { test: true }); - await queue.add('delayed-job', { test: true }, { delay: 60000 }); - - const metrics = await metricsCollector.collect(); - - expect(metrics.waiting).toBe(1); - expect(metrics.delayed).toBe(1); - expect(metrics.active).toBe(0); - expect(metrics.completed).toBe(0); - expect(metrics.failed).toBe(0); - }); - - test('should track completed and failed jobs', async () => { - let jobCount = 0; - - // Create worker that alternates between success and failure - worker = new Worker('metrics-test-queue', async () => { - jobCount++; - if (jobCount % 2 === 0) { - throw new Error('Test failure'); - } - return { success: true }; - }, { connection }); - - // Add jobs - await queue.add('job1', { test: 1 }); - await queue.add('job2', { test: 2 }); - await queue.add('job3', { test: 3 }); - await queue.add('job4', { test: 4 }); - - // Wait for processing - await new Promise(resolve => setTimeout(resolve, 200)); - - const metrics = await metricsCollector.collect(); - - expect(metrics.completed).toBe(2); - expect(metrics.failed).toBe(2); - }); - }); - - describe('Processing Time Metrics', () => { - test('should track processing times', async () => { - const processingTimes = [50, 100, 150, 200, 250]; - let jobIndex = 0; - - // Create worker with variable processing times - worker = new Worker('metrics-test-queue', async () => { - const delay = processingTimes[jobIndex++] || 100; - await new Promise(resolve => setTimeout(resolve, delay)); - return { processed: true }; - }, { connection }); - - // Add jobs - for (let i = 0; i < processingTimes.length; i++) { - await queue.add(`job${i}`, { index: i }); - } - - // Wait for processing - await new Promise(resolve => setTimeout(resolve, 1500)); - - const metrics = await metricsCollector.collect(); - - expect(metrics.processingTime.avg).toBeGreaterThan(0); - expect(metrics.processingTime.min).toBeGreaterThanOrEqual(50); - expect(metrics.processingTime.max).toBeLessThanOrEqual(300); - expect(metrics.processingTime.p95).toBeGreaterThan(metrics.processingTime.avg); - }); - - test('should handle empty processing times', async () => { - const metrics = await metricsCollector.collect(); - - expect(metrics.processingTime).toEqual({ - avg: 0, - min: 0, - max: 0, - p95: 0, - p99: 0, - }); - }); - }); - - describe('Throughput Metrics', () => { - test('should calculate throughput correctly', async () => { - // Create fast worker - worker = new Worker('metrics-test-queue', async () => { - return { success: true }; - }, { connection, concurrency: 5 }); - - // Add multiple jobs - const jobPromises = []; - for (let i = 0; i < 10; i++) { - jobPromises.push(queue.add(`job${i}`, { index: i })); - } - await Promise.all(jobPromises); - - // Wait for processing - await new Promise(resolve => setTimeout(resolve, 500)); - - const metrics = await metricsCollector.collect(); - - expect(metrics.throughput.completedPerMinute).toBeGreaterThan(0); - expect(metrics.throughput.totalPerMinute).toBe( - metrics.throughput.completedPerMinute + metrics.throughput.failedPerMinute - ); - }); - }); - - describe('Queue Health', () => { - test('should report healthy queue', async () => { - const metrics = await metricsCollector.collect(); - - expect(metrics.isHealthy).toBe(true); - expect(metrics.healthIssues).toEqual([]); - }); - - test('should detect high failure rate', async () => { - // Create worker that always fails - worker = new Worker('metrics-test-queue', async () => { - throw new Error('Always fails'); - }, { connection }); - - // Add jobs - for (let i = 0; i < 10; i++) { - await queue.add(`job${i}`, { index: i }); - } - - // Wait for failures - await new Promise(resolve => setTimeout(resolve, 500)); - - const metrics = await metricsCollector.collect(); - - expect(metrics.isHealthy).toBe(false); - expect(metrics.healthIssues).toContain( - expect.stringMatching(/High failure rate/) - ); - }); - - test('should detect large queue backlog', async () => { - // Add many jobs without workers - for (let i = 0; i < 1001; i++) { - await queue.add(`job${i}`, { index: i }); - } - - const metrics = await metricsCollector.collect(); - - expect(metrics.isHealthy).toBe(false); - expect(metrics.healthIssues).toContain( - expect.stringMatching(/Large queue backlog/) - ); - }); - }); - - describe('Oldest Waiting Job', () => { - test('should track oldest waiting job', async () => { - const beforeAdd = Date.now(); - - // Add jobs with delays - await queue.add('old-job', { test: true }); - await new Promise(resolve => setTimeout(resolve, 100)); - await queue.add('new-job', { test: true }); - - const metrics = await metricsCollector.collect(); - - expect(metrics.oldestWaitingJob).toBeDefined(); - expect(metrics.oldestWaitingJob!.getTime()).toBeGreaterThanOrEqual(beforeAdd); - }); - - test('should return null when no waiting jobs', async () => { - // Create worker that processes immediately - worker = new Worker('metrics-test-queue', async () => { - return { success: true }; - }, { connection }); - - const metrics = await metricsCollector.collect(); - expect(metrics.oldestWaitingJob).toBe(null); - }); - }); - - describe('Metrics Report', () => { - test('should generate formatted report', async () => { - // Add some jobs - await queue.add('job1', { test: true }); - await queue.add('job2', { test: true }, { delay: 5000 }); - - const report = await metricsCollector.getReport(); - - expect(report).toContain('Queue Metrics Report'); - expect(report).toContain('Status:'); - expect(report).toContain('Job Counts:'); - expect(report).toContain('Performance:'); - expect(report).toContain('Throughput:'); - expect(report).toContain('Waiting: 1'); - expect(report).toContain('Delayed: 1'); - }); - - test('should include health issues in report', async () => { - // Add many jobs to trigger health issue - for (let i = 0; i < 1001; i++) { - await queue.add(`job${i}`, { index: i }); - } - - const report = await metricsCollector.getReport(); - - expect(report).toContain('Issues Detected'); - expect(report).toContain('Health Issues:'); - expect(report).toContain('Large queue backlog'); - }); - }); - - describe('Prometheus Metrics', () => { - test('should export metrics in Prometheus format', async () => { - // Add some jobs and process them - worker = new Worker('metrics-test-queue', async () => { - await new Promise(resolve => setTimeout(resolve, 50)); - return { success: true }; - }, { connection }); - - await queue.add('job1', { test: true }); - await queue.add('job2', { test: true }); - - // Wait for processing - await new Promise(resolve => setTimeout(resolve, 200)); - - const prometheusMetrics = await metricsCollector.getPrometheusMetrics(); - - // Check format - expect(prometheusMetrics).toContain('# HELP queue_jobs_total'); - expect(prometheusMetrics).toContain('# TYPE queue_jobs_total gauge'); - expect(prometheusMetrics).toContain('queue_jobs_total{queue="metrics-test-queue",status="completed"}'); - - expect(prometheusMetrics).toContain('# HELP queue_processing_time_seconds'); - expect(prometheusMetrics).toContain('# TYPE queue_processing_time_seconds summary'); - - expect(prometheusMetrics).toContain('# HELP queue_throughput_per_minute'); - expect(prometheusMetrics).toContain('# TYPE queue_throughput_per_minute gauge'); - - expect(prometheusMetrics).toContain('# HELP queue_health'); - expect(prometheusMetrics).toContain('# TYPE queue_health gauge'); - }); - }); -}); \ No newline at end of file +import { Queue, QueueEvents, Worker } from 'bullmq'; +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { QueueMetricsCollector } from '../src/queue-metrics'; +import { getRedisConnection } from '../src/utils'; + +// Suppress Redis connection errors in tests +process.on('unhandledRejection', (reason, promise) => { + if (reason && typeof reason === 'object' && 'message' in reason) { + const message = (reason as Error).message; + if ( + message.includes('Connection is closed') || + message.includes('Connection is in monitoring mode') + ) { + return; + } + } + console.error('Unhandled Rejection at:', promise, 'reason:', reason); +}); + +describe('QueueMetricsCollector', () => { + let queue: Queue; + let queueEvents: QueueEvents; + let metricsCollector: QueueMetricsCollector; + let worker: Worker; + let connection: any; + + const redisConfig = { + host: 'localhost', + port: 6379, + password: '', + db: 0, + }; + + beforeEach(async () => { + connection = getRedisConnection(redisConfig); + + // Create queue and events + queue = new Queue('metrics-test-queue', { connection }); + queueEvents = new QueueEvents('metrics-test-queue', { connection }); + + // Create metrics collector + metricsCollector = new QueueMetricsCollector(queue, queueEvents); + + // Wait for connections + await queue.waitUntilReady(); + await queueEvents.waitUntilReady(); + }); + + afterEach(async () => { + try { + if (worker) { + await worker.close(); + } + await queueEvents.close(); + await queue.close(); + } catch { + // Ignore cleanup errors + } + await new Promise(resolve => setTimeout(resolve, 50)); + }); + + describe('Job Count Metrics', () => { + test('should collect basic job counts', async () => { + // Add jobs in different states + await queue.add('waiting-job', { test: true }); + await queue.add('delayed-job', { test: true }, { delay: 60000 }); + + const metrics = await metricsCollector.collect(); + + expect(metrics.waiting).toBe(1); + expect(metrics.delayed).toBe(1); + expect(metrics.active).toBe(0); + expect(metrics.completed).toBe(0); + expect(metrics.failed).toBe(0); + }); + + test('should track completed and failed jobs', async () => { + let jobCount = 0; + + // Create worker that alternates between success and failure + worker = new Worker( + 'metrics-test-queue', + async () => { + jobCount++; + if (jobCount % 2 === 0) { + throw new Error('Test failure'); + } + return { success: true }; + }, + { connection } + ); + + // Add jobs + await queue.add('job1', { test: 1 }); + await queue.add('job2', { test: 2 }); + await queue.add('job3', { test: 3 }); + await queue.add('job4', { test: 4 }); + + // Wait for processing + await new Promise(resolve => setTimeout(resolve, 200)); + + const metrics = await metricsCollector.collect(); + + expect(metrics.completed).toBe(2); + expect(metrics.failed).toBe(2); + }); + }); + + describe('Processing Time Metrics', () => { + test('should track processing times', async () => { + const processingTimes = [50, 100, 150, 200, 250]; + let jobIndex = 0; + + // Create worker with variable processing times + worker = new Worker( + 'metrics-test-queue', + async () => { + const delay = processingTimes[jobIndex++] || 100; + await new Promise(resolve => setTimeout(resolve, delay)); + return { processed: true }; + }, + { connection } + ); + + // Add jobs + for (let i = 0; i < processingTimes.length; i++) { + await queue.add(`job${i}`, { index: i }); + } + + // Wait for processing + await new Promise(resolve => setTimeout(resolve, 1500)); + + const metrics = await metricsCollector.collect(); + + expect(metrics.processingTime.avg).toBeGreaterThan(0); + expect(metrics.processingTime.min).toBeGreaterThanOrEqual(50); + expect(metrics.processingTime.max).toBeLessThanOrEqual(300); + expect(metrics.processingTime.p95).toBeGreaterThan(metrics.processingTime.avg); + }); + + test('should handle empty processing times', async () => { + const metrics = await metricsCollector.collect(); + + expect(metrics.processingTime).toEqual({ + avg: 0, + min: 0, + max: 0, + p95: 0, + p99: 0, + }); + }); + }); + + describe('Throughput Metrics', () => { + test('should calculate throughput correctly', async () => { + // Create fast worker + worker = new Worker( + 'metrics-test-queue', + async () => { + return { success: true }; + }, + { connection, concurrency: 5 } + ); + + // Add multiple jobs + const jobPromises = []; + for (let i = 0; i < 10; i++) { + jobPromises.push(queue.add(`job${i}`, { index: i })); + } + await Promise.all(jobPromises); + + // Wait for processing + await new Promise(resolve => setTimeout(resolve, 500)); + + const metrics = await metricsCollector.collect(); + + expect(metrics.throughput.completedPerMinute).toBeGreaterThan(0); + expect(metrics.throughput.totalPerMinute).toBe( + metrics.throughput.completedPerMinute + metrics.throughput.failedPerMinute + ); + }); + }); + + describe('Queue Health', () => { + test('should report healthy queue', async () => { + const metrics = await metricsCollector.collect(); + + expect(metrics.isHealthy).toBe(true); + expect(metrics.healthIssues).toEqual([]); + }); + + test('should detect high failure rate', async () => { + // Create worker that always fails + worker = new Worker( + 'metrics-test-queue', + async () => { + throw new Error('Always fails'); + }, + { connection } + ); + + // Add jobs + for (let i = 0; i < 10; i++) { + await queue.add(`job${i}`, { index: i }); + } + + // Wait for failures + await new Promise(resolve => setTimeout(resolve, 500)); + + const metrics = await metricsCollector.collect(); + + expect(metrics.isHealthy).toBe(false); + expect(metrics.healthIssues).toContain(expect.stringMatching(/High failure rate/)); + }); + + test('should detect large queue backlog', async () => { + // Add many jobs without workers + for (let i = 0; i < 1001; i++) { + await queue.add(`job${i}`, { index: i }); + } + + const metrics = await metricsCollector.collect(); + + expect(metrics.isHealthy).toBe(false); + expect(metrics.healthIssues).toContain(expect.stringMatching(/Large queue backlog/)); + }); + }); + + describe('Oldest Waiting Job', () => { + test('should track oldest waiting job', async () => { + const beforeAdd = Date.now(); + + // Add jobs with delays + await queue.add('old-job', { test: true }); + await new Promise(resolve => setTimeout(resolve, 100)); + await queue.add('new-job', { test: true }); + + const metrics = await metricsCollector.collect(); + + expect(metrics.oldestWaitingJob).toBeDefined(); + expect(metrics.oldestWaitingJob!.getTime()).toBeGreaterThanOrEqual(beforeAdd); + }); + + test('should return null when no waiting jobs', async () => { + // Create worker that processes immediately + worker = new Worker( + 'metrics-test-queue', + async () => { + return { success: true }; + }, + { connection } + ); + + const metrics = await metricsCollector.collect(); + expect(metrics.oldestWaitingJob).toBe(null); + }); + }); + + describe('Metrics Report', () => { + test('should generate formatted report', async () => { + // Add some jobs + await queue.add('job1', { test: true }); + await queue.add('job2', { test: true }, { delay: 5000 }); + + const report = await metricsCollector.getReport(); + + expect(report).toContain('Queue Metrics Report'); + expect(report).toContain('Status:'); + expect(report).toContain('Job Counts:'); + expect(report).toContain('Performance:'); + expect(report).toContain('Throughput:'); + expect(report).toContain('Waiting: 1'); + expect(report).toContain('Delayed: 1'); + }); + + test('should include health issues in report', async () => { + // Add many jobs to trigger health issue + for (let i = 0; i < 1001; i++) { + await queue.add(`job${i}`, { index: i }); + } + + const report = await metricsCollector.getReport(); + + expect(report).toContain('Issues Detected'); + expect(report).toContain('Health Issues:'); + expect(report).toContain('Large queue backlog'); + }); + }); + + describe('Prometheus Metrics', () => { + test('should export metrics in Prometheus format', async () => { + // Add some jobs and process them + worker = new Worker( + 'metrics-test-queue', + async () => { + await new Promise(resolve => setTimeout(resolve, 50)); + return { success: true }; + }, + { connection } + ); + + await queue.add('job1', { test: true }); + await queue.add('job2', { test: true }); + + // Wait for processing + await new Promise(resolve => setTimeout(resolve, 200)); + + const prometheusMetrics = await metricsCollector.getPrometheusMetrics(); + + // Check format + expect(prometheusMetrics).toContain('# HELP queue_jobs_total'); + expect(prometheusMetrics).toContain('# TYPE queue_jobs_total gauge'); + expect(prometheusMetrics).toContain( + 'queue_jobs_total{queue="metrics-test-queue",status="completed"}' + ); + + expect(prometheusMetrics).toContain('# HELP queue_processing_time_seconds'); + expect(prometheusMetrics).toContain('# TYPE queue_processing_time_seconds summary'); + + expect(prometheusMetrics).toContain('# HELP queue_throughput_per_minute'); + expect(prometheusMetrics).toContain('# TYPE queue_throughput_per_minute gauge'); + + expect(prometheusMetrics).toContain('# HELP queue_health'); + expect(prometheusMetrics).toContain('# TYPE queue_health gauge'); + }); + }); +}); diff --git a/libs/services/queue/test/queue-simple.test.ts b/libs/services/queue/test/queue-simple.test.ts index 2820c21..31f14e8 100644 --- a/libs/services/queue/test/queue-simple.test.ts +++ b/libs/services/queue/test/queue-simple.test.ts @@ -1,81 +1,81 @@ -import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; -import { QueueManager, handlerRegistry } from '../src'; - -describe('QueueManager Simple Tests', () => { - let queueManager: QueueManager; - - // Assumes Redis is running locally on default port - const redisConfig = { - host: 'localhost', - port: 6379, - }; - - beforeEach(() => { - handlerRegistry.clear(); - }); - - afterEach(async () => { - if (queueManager) { - try { - await queueManager.shutdown(); - } catch { - // Ignore errors during cleanup - } - } - }); - - test('should create queue manager instance', () => { - queueManager = new QueueManager({ - queueName: 'test-queue', - redis: redisConfig, - }); - - expect(queueManager.queueName).toBe('test-queue'); - }); - - test('should handle missing Redis gracefully', async () => { - // Use a port that's likely not running Redis - queueManager = new QueueManager({ - queueName: 'test-queue', - redis: { - host: 'localhost', - port: 9999, - }, - }); - - await expect(queueManager.initialize()).rejects.toThrow(); - }); - - test('handler registry should work', () => { - const testHandler = async (payload: any) => { - return { success: true, payload }; - }; - - handlerRegistry.register('test-handler', { - 'test-op': testHandler, - }); - - const handler = handlerRegistry.getHandler('test-handler', 'test-op'); - expect(handler).toBe(testHandler); - }); - - test('handler registry should return null for missing handler', () => { - const handler = handlerRegistry.getHandler('missing', 'op'); - expect(handler).toBe(null); - }); - - test('should get handler statistics', () => { - handlerRegistry.register('handler1', { - 'op1': async () => ({}), - 'op2': async () => ({}), - }); - - handlerRegistry.register('handler2', { - 'op1': async () => ({}), - }); - - const stats = handlerRegistry.getStats(); - expect(stats.handlers).toBe(2); - expect(stats.totalOperations).toBe(3); - }); -}); \ No newline at end of file +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { handlerRegistry, QueueManager } from '../src'; + +describe('QueueManager Simple Tests', () => { + let queueManager: QueueManager; + + // Assumes Redis is running locally on default port + const redisConfig = { + host: 'localhost', + port: 6379, + }; + + beforeEach(() => { + handlerRegistry.clear(); + }); + + afterEach(async () => { + if (queueManager) { + try { + await queueManager.shutdown(); + } catch { + // Ignore errors during cleanup + } + } + }); + + test('should create queue manager instance', () => { + queueManager = new QueueManager({ + queueName: 'test-queue', + redis: redisConfig, + }); + + expect(queueManager.queueName).toBe('test-queue'); + }); + + test('should handle missing Redis gracefully', async () => { + // Use a port that's likely not running Redis + queueManager = new QueueManager({ + queueName: 'test-queue', + redis: { + host: 'localhost', + port: 9999, + }, + }); + + await expect(queueManager.initialize()).rejects.toThrow(); + }); + + test('handler registry should work', () => { + const testHandler = async (payload: any) => { + return { success: true, payload }; + }; + + handlerRegistry.register('test-handler', { + 'test-op': testHandler, + }); + + const handler = handlerRegistry.getHandler('test-handler', 'test-op'); + expect(handler).toBe(testHandler); + }); + + test('handler registry should return null for missing handler', () => { + const handler = handlerRegistry.getHandler('missing', 'op'); + expect(handler).toBe(null); + }); + + test('should get handler statistics', () => { + handlerRegistry.register('handler1', { + op1: async () => ({}), + op2: async () => ({}), + }); + + handlerRegistry.register('handler2', { + op1: async () => ({}), + }); + + const stats = handlerRegistry.getStats(); + expect(stats.handlers).toBe(2); + expect(stats.totalOperations).toBe(3); + }); +}); diff --git a/libs/services/queue/test/rate-limiter.test.ts b/libs/services/queue/test/rate-limiter.test.ts index 0007abb..255de6e 100644 --- a/libs/services/queue/test/rate-limiter.test.ts +++ b/libs/services/queue/test/rate-limiter.test.ts @@ -1,309 +1,311 @@ -import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; -import { QueueRateLimiter } from '../src/rate-limiter'; -import { getRedisConnection } from '../src/utils'; -import Redis from 'ioredis'; - -// Suppress Redis connection errors in tests -process.on('unhandledRejection', (reason, promise) => { - if (reason && typeof reason === 'object' && 'message' in reason) { - const message = (reason as Error).message; - if (message.includes('Connection is closed') || - message.includes('Connection is in monitoring mode')) { - return; - } - } - console.error('Unhandled Rejection at:', promise, 'reason:', reason); -}); - -describe('QueueRateLimiter', () => { - let redisClient: Redis; - let rateLimiter: QueueRateLimiter; - - const redisConfig = { - host: 'localhost', - port: 6379, - password: '', - db: 0, - }; - - beforeEach(async () => { - // Create Redis client - redisClient = new Redis(getRedisConnection(redisConfig)); - - // Clear Redis keys for tests - try { - const keys = await redisClient.keys('rl:*'); - if (keys.length > 0) { - await redisClient.del(...keys); - } - } catch { - // Ignore cleanup errors - } - rateLimiter = new QueueRateLimiter(redisClient); - }); - - afterEach(async () => { - if (redisClient) { - try { - await redisClient.quit(); - } catch { - // Ignore cleanup errors - } - } - await new Promise(resolve => setTimeout(resolve, 50)); - }); - - describe('Rate Limit Rules', () => { - test('should add and enforce global rate limit', async () => { - rateLimiter.addRule({ - level: 'global', - config: { - points: 5, - duration: 1, // 1 second - }, - }); - - // Consume 5 points - for (let i = 0; i < 5; i++) { - const result = await rateLimiter.checkLimit('any-handler', 'any-operation'); - expect(result.allowed).toBe(true); - } - - // 6th request should be blocked - const blocked = await rateLimiter.checkLimit('any-handler', 'any-operation'); - expect(blocked.allowed).toBe(false); - expect(blocked.retryAfter).toBeGreaterThan(0); - }); - - test('should add and enforce handler-level rate limit', async () => { - rateLimiter.addRule({ - level: 'handler', - handler: 'api-handler', - config: { - points: 3, - duration: 1, - }, - }); - - // api-handler should be limited - for (let i = 0; i < 3; i++) { - const result = await rateLimiter.checkLimit('api-handler', 'any-operation'); - expect(result.allowed).toBe(true); - } - - const blocked = await rateLimiter.checkLimit('api-handler', 'any-operation'); - expect(blocked.allowed).toBe(false); - - // Other handlers should not be limited - const otherHandler = await rateLimiter.checkLimit('other-handler', 'any-operation'); - expect(otherHandler.allowed).toBe(true); - }); - - test('should add and enforce operation-level rate limit', async () => { - rateLimiter.addRule({ - level: 'operation', - handler: 'data-handler', - operation: 'fetch-prices', - config: { - points: 2, - duration: 1, - }, - }); - - // Specific operation should be limited - for (let i = 0; i < 2; i++) { - const result = await rateLimiter.checkLimit('data-handler', 'fetch-prices'); - expect(result.allowed).toBe(true); - } - - const blocked = await rateLimiter.checkLimit('data-handler', 'fetch-prices'); - expect(blocked.allowed).toBe(false); - - // Other operations on same handler should work - const otherOp = await rateLimiter.checkLimit('data-handler', 'fetch-volume'); - expect(otherOp.allowed).toBe(true); - }); - - test('should enforce multiple rate limits (most restrictive wins)', async () => { - // Global: 10/sec - rateLimiter.addRule({ - level: 'global', - config: { points: 10, duration: 1 }, - }); - - // Handler: 5/sec - rateLimiter.addRule({ - level: 'handler', - handler: 'test-handler', - config: { points: 5, duration: 1 }, - }); - - // Operation: 2/sec - rateLimiter.addRule({ - level: 'operation', - handler: 'test-handler', - operation: 'test-op', - config: { points: 2, duration: 1 }, - }); - - // Should be limited by operation level (most restrictive) - for (let i = 0; i < 2; i++) { - const result = await rateLimiter.checkLimit('test-handler', 'test-op'); - expect(result.allowed).toBe(true); - } - - const blocked = await rateLimiter.checkLimit('test-handler', 'test-op'); - expect(blocked.allowed).toBe(false); - }); - }); - - describe('Rate Limit Status', () => { - test('should get rate limit status', async () => { - rateLimiter.addRule({ - level: 'handler', - handler: 'status-test', - config: { points: 10, duration: 60 }, - }); - - // Consume some points - await rateLimiter.checkLimit('status-test', 'operation'); - await rateLimiter.checkLimit('status-test', 'operation'); - - const status = await rateLimiter.getStatus('status-test', 'operation'); - expect(status.handler).toBe('status-test'); - expect(status.operation).toBe('operation'); - expect(status.limits.length).toBe(1); - expect(status.limits[0].points).toBe(10); - expect(status.limits[0].remaining).toBe(8); - }); - - test('should show multiple applicable limits in status', async () => { - rateLimiter.addRule({ - level: 'global', - config: { points: 100, duration: 60 }, - }); - - rateLimiter.addRule({ - level: 'handler', - handler: 'multi-test', - config: { points: 50, duration: 60 }, - }); - - const status = await rateLimiter.getStatus('multi-test', 'operation'); - expect(status.limits.length).toBe(2); - - const globalLimit = status.limits.find(l => l.level === 'global'); - const handlerLimit = status.limits.find(l => l.level === 'handler'); - - expect(globalLimit?.points).toBe(100); - expect(handlerLimit?.points).toBe(50); - }); - }); - - describe('Rate Limit Management', () => { - test('should reset rate limits', async () => { - rateLimiter.addRule({ - level: 'handler', - handler: 'reset-test', - config: { points: 1, duration: 60 }, - }); - - // Consume the limit - await rateLimiter.checkLimit('reset-test', 'operation'); - const blocked = await rateLimiter.checkLimit('reset-test', 'operation'); - expect(blocked.allowed).toBe(false); - - // Reset limits - await rateLimiter.reset('reset-test'); - - // Should be allowed again - const afterReset = await rateLimiter.checkLimit('reset-test', 'operation'); - expect(afterReset.allowed).toBe(true); - }); - - test('should get all rules', async () => { - rateLimiter.addRule({ - level: 'global', - config: { points: 100, duration: 60 }, - }); - - rateLimiter.addRule({ - level: 'handler', - handler: 'test', - config: { points: 50, duration: 60 }, - }); - - const rules = rateLimiter.getRules(); - expect(rules.length).toBe(2); - expect(rules[0].level).toBe('global'); - expect(rules[1].level).toBe('handler'); - }); - - test('should remove specific rule', async () => { - rateLimiter.addRule({ - level: 'handler', - handler: 'remove-test', - config: { points: 1, duration: 1 }, - }); - - // Verify rule exists - await rateLimiter.checkLimit('remove-test', 'op'); - const blocked = await rateLimiter.checkLimit('remove-test', 'op'); - expect(blocked.allowed).toBe(false); - - // Remove rule - const removed = rateLimiter.removeRule('handler', 'remove-test'); - expect(removed).toBe(true); - - // Should not be limited anymore - const afterRemove = await rateLimiter.checkLimit('remove-test', 'op'); - expect(afterRemove.allowed).toBe(true); - }); - }); - - describe('Block Duration', () => { - test('should block for specified duration after limit exceeded', async () => { - rateLimiter.addRule({ - level: 'handler', - handler: 'block-test', - config: { - points: 1, - duration: 1, - blockDuration: 2, // Block for 2 seconds - }, - }); - - // Consume limit - await rateLimiter.checkLimit('block-test', 'op'); - - // Should be blocked - const blocked = await rateLimiter.checkLimit('block-test', 'op'); - expect(blocked.allowed).toBe(false); - expect(blocked.retryAfter).toBeGreaterThanOrEqual(1000); // At least 1 second - }); - }); - - describe('Error Handling', () => { - test('should allow requests when rate limiter fails', async () => { - // Create a rate limiter with invalid redis client - const badRedis = new Redis({ - host: 'invalid-host', - port: 9999, - retryStrategy: () => null, // Disable retries - }); - - const failingLimiter = new QueueRateLimiter(badRedis); - - failingLimiter.addRule({ - level: 'global', - config: { points: 1, duration: 1 }, - }); - - // Should allow even though Redis is not available - const result = await failingLimiter.checkLimit('test', 'test'); - expect(result.allowed).toBe(true); - - badRedis.disconnect(); - }); - }); -}); \ No newline at end of file +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import Redis from 'ioredis'; +import { QueueRateLimiter } from '../src/rate-limiter'; +import { getRedisConnection } from '../src/utils'; + +// Suppress Redis connection errors in tests +process.on('unhandledRejection', (reason, promise) => { + if (reason && typeof reason === 'object' && 'message' in reason) { + const message = (reason as Error).message; + if ( + message.includes('Connection is closed') || + message.includes('Connection is in monitoring mode') + ) { + return; + } + } + console.error('Unhandled Rejection at:', promise, 'reason:', reason); +}); + +describe('QueueRateLimiter', () => { + let redisClient: Redis; + let rateLimiter: QueueRateLimiter; + + const redisConfig = { + host: 'localhost', + port: 6379, + password: '', + db: 0, + }; + + beforeEach(async () => { + // Create Redis client + redisClient = new Redis(getRedisConnection(redisConfig)); + + // Clear Redis keys for tests + try { + const keys = await redisClient.keys('rl:*'); + if (keys.length > 0) { + await redisClient.del(...keys); + } + } catch { + // Ignore cleanup errors + } + rateLimiter = new QueueRateLimiter(redisClient); + }); + + afterEach(async () => { + if (redisClient) { + try { + await redisClient.quit(); + } catch { + // Ignore cleanup errors + } + } + await new Promise(resolve => setTimeout(resolve, 50)); + }); + + describe('Rate Limit Rules', () => { + test('should add and enforce global rate limit', async () => { + rateLimiter.addRule({ + level: 'global', + config: { + points: 5, + duration: 1, // 1 second + }, + }); + + // Consume 5 points + for (let i = 0; i < 5; i++) { + const result = await rateLimiter.checkLimit('any-handler', 'any-operation'); + expect(result.allowed).toBe(true); + } + + // 6th request should be blocked + const blocked = await rateLimiter.checkLimit('any-handler', 'any-operation'); + expect(blocked.allowed).toBe(false); + expect(blocked.retryAfter).toBeGreaterThan(0); + }); + + test('should add and enforce handler-level rate limit', async () => { + rateLimiter.addRule({ + level: 'handler', + handler: 'api-handler', + config: { + points: 3, + duration: 1, + }, + }); + + // api-handler should be limited + for (let i = 0; i < 3; i++) { + const result = await rateLimiter.checkLimit('api-handler', 'any-operation'); + expect(result.allowed).toBe(true); + } + + const blocked = await rateLimiter.checkLimit('api-handler', 'any-operation'); + expect(blocked.allowed).toBe(false); + + // Other handlers should not be limited + const otherHandler = await rateLimiter.checkLimit('other-handler', 'any-operation'); + expect(otherHandler.allowed).toBe(true); + }); + + test('should add and enforce operation-level rate limit', async () => { + rateLimiter.addRule({ + level: 'operation', + handler: 'data-handler', + operation: 'fetch-prices', + config: { + points: 2, + duration: 1, + }, + }); + + // Specific operation should be limited + for (let i = 0; i < 2; i++) { + const result = await rateLimiter.checkLimit('data-handler', 'fetch-prices'); + expect(result.allowed).toBe(true); + } + + const blocked = await rateLimiter.checkLimit('data-handler', 'fetch-prices'); + expect(blocked.allowed).toBe(false); + + // Other operations on same handler should work + const otherOp = await rateLimiter.checkLimit('data-handler', 'fetch-volume'); + expect(otherOp.allowed).toBe(true); + }); + + test('should enforce multiple rate limits (most restrictive wins)', async () => { + // Global: 10/sec + rateLimiter.addRule({ + level: 'global', + config: { points: 10, duration: 1 }, + }); + + // Handler: 5/sec + rateLimiter.addRule({ + level: 'handler', + handler: 'test-handler', + config: { points: 5, duration: 1 }, + }); + + // Operation: 2/sec + rateLimiter.addRule({ + level: 'operation', + handler: 'test-handler', + operation: 'test-op', + config: { points: 2, duration: 1 }, + }); + + // Should be limited by operation level (most restrictive) + for (let i = 0; i < 2; i++) { + const result = await rateLimiter.checkLimit('test-handler', 'test-op'); + expect(result.allowed).toBe(true); + } + + const blocked = await rateLimiter.checkLimit('test-handler', 'test-op'); + expect(blocked.allowed).toBe(false); + }); + }); + + describe('Rate Limit Status', () => { + test('should get rate limit status', async () => { + rateLimiter.addRule({ + level: 'handler', + handler: 'status-test', + config: { points: 10, duration: 60 }, + }); + + // Consume some points + await rateLimiter.checkLimit('status-test', 'operation'); + await rateLimiter.checkLimit('status-test', 'operation'); + + const status = await rateLimiter.getStatus('status-test', 'operation'); + expect(status.handler).toBe('status-test'); + expect(status.operation).toBe('operation'); + expect(status.limits.length).toBe(1); + expect(status.limits[0].points).toBe(10); + expect(status.limits[0].remaining).toBe(8); + }); + + test('should show multiple applicable limits in status', async () => { + rateLimiter.addRule({ + level: 'global', + config: { points: 100, duration: 60 }, + }); + + rateLimiter.addRule({ + level: 'handler', + handler: 'multi-test', + config: { points: 50, duration: 60 }, + }); + + const status = await rateLimiter.getStatus('multi-test', 'operation'); + expect(status.limits.length).toBe(2); + + const globalLimit = status.limits.find(l => l.level === 'global'); + const handlerLimit = status.limits.find(l => l.level === 'handler'); + + expect(globalLimit?.points).toBe(100); + expect(handlerLimit?.points).toBe(50); + }); + }); + + describe('Rate Limit Management', () => { + test('should reset rate limits', async () => { + rateLimiter.addRule({ + level: 'handler', + handler: 'reset-test', + config: { points: 1, duration: 60 }, + }); + + // Consume the limit + await rateLimiter.checkLimit('reset-test', 'operation'); + const blocked = await rateLimiter.checkLimit('reset-test', 'operation'); + expect(blocked.allowed).toBe(false); + + // Reset limits + await rateLimiter.reset('reset-test'); + + // Should be allowed again + const afterReset = await rateLimiter.checkLimit('reset-test', 'operation'); + expect(afterReset.allowed).toBe(true); + }); + + test('should get all rules', async () => { + rateLimiter.addRule({ + level: 'global', + config: { points: 100, duration: 60 }, + }); + + rateLimiter.addRule({ + level: 'handler', + handler: 'test', + config: { points: 50, duration: 60 }, + }); + + const rules = rateLimiter.getRules(); + expect(rules.length).toBe(2); + expect(rules[0].level).toBe('global'); + expect(rules[1].level).toBe('handler'); + }); + + test('should remove specific rule', async () => { + rateLimiter.addRule({ + level: 'handler', + handler: 'remove-test', + config: { points: 1, duration: 1 }, + }); + + // Verify rule exists + await rateLimiter.checkLimit('remove-test', 'op'); + const blocked = await rateLimiter.checkLimit('remove-test', 'op'); + expect(blocked.allowed).toBe(false); + + // Remove rule + const removed = rateLimiter.removeRule('handler', 'remove-test'); + expect(removed).toBe(true); + + // Should not be limited anymore + const afterRemove = await rateLimiter.checkLimit('remove-test', 'op'); + expect(afterRemove.allowed).toBe(true); + }); + }); + + describe('Block Duration', () => { + test('should block for specified duration after limit exceeded', async () => { + rateLimiter.addRule({ + level: 'handler', + handler: 'block-test', + config: { + points: 1, + duration: 1, + blockDuration: 2, // Block for 2 seconds + }, + }); + + // Consume limit + await rateLimiter.checkLimit('block-test', 'op'); + + // Should be blocked + const blocked = await rateLimiter.checkLimit('block-test', 'op'); + expect(blocked.allowed).toBe(false); + expect(blocked.retryAfter).toBeGreaterThanOrEqual(1000); // At least 1 second + }); + }); + + describe('Error Handling', () => { + test('should allow requests when rate limiter fails', async () => { + // Create a rate limiter with invalid redis client + const badRedis = new Redis({ + host: 'invalid-host', + port: 9999, + retryStrategy: () => null, // Disable retries + }); + + const failingLimiter = new QueueRateLimiter(badRedis); + + failingLimiter.addRule({ + level: 'global', + config: { points: 1, duration: 1 }, + }); + + // Should allow even though Redis is not available + const result = await failingLimiter.checkLimit('test', 'test'); + expect(result.allowed).toBe(true); + + badRedis.disconnect(); + }); + }); +}); diff --git a/libs/services/shutdown/src/index.ts b/libs/services/shutdown/src/index.ts index b498a06..14319cd 100644 --- a/libs/services/shutdown/src/index.ts +++ b/libs/services/shutdown/src/index.ts @@ -9,7 +9,12 @@ import type { ShutdownResult } from './types'; // Core shutdown classes and types export { Shutdown } from './shutdown'; -export type { ShutdownCallback, ShutdownOptions, ShutdownResult, PrioritizedShutdownCallback } from './types'; +export type { + ShutdownCallback, + ShutdownOptions, + ShutdownResult, + PrioritizedShutdownCallback, +} from './types'; // Global singleton instance let globalInstance: Shutdown | null = null; @@ -31,7 +36,11 @@ function getGlobalInstance(): Shutdown { /** * Register a cleanup callback that will be executed during shutdown */ -export function onShutdown(callback: () => Promise | void, priority?: number, name?: string): void { +export function onShutdown( + callback: () => Promise | void, + priority?: number, + name?: string +): void { getGlobalInstance().onShutdown(callback, priority, name); } diff --git a/libs/services/shutdown/src/shutdown.ts b/libs/services/shutdown/src/shutdown.ts index 78a0be8..36e0276 100644 --- a/libs/services/shutdown/src/shutdown.ts +++ b/libs/services/shutdown/src/shutdown.ts @@ -8,7 +8,12 @@ * - Platform-specific signal support (Windows/Unix) */ -import type { PrioritizedShutdownCallback, ShutdownCallback, ShutdownOptions, ShutdownResult } from './types'; +import type { + PrioritizedShutdownCallback, + ShutdownCallback, + ShutdownOptions, + ShutdownResult, +} from './types'; // Global flag that works across all processes/workers declare global { diff --git a/libs/services/shutdown/tsconfig.json b/libs/services/shutdown/tsconfig.json index dbc9566..9405533 100644 --- a/libs/services/shutdown/tsconfig.json +++ b/libs/services/shutdown/tsconfig.json @@ -6,6 +6,5 @@ "composite": true }, "include": ["src/**/*"], - "references": [ - ] + "references": [] } diff --git a/libs/utils/package.json b/libs/utils/package.json index 2c076d1..e11b506 100644 --- a/libs/utils/package.json +++ b/libs/utils/package.json @@ -20,4 +20,4 @@ "typescript": "^5.3.0", "bun-types": "^1.2.15" } -} \ No newline at end of file +} diff --git a/libs/utils/src/calculations/index.ts b/libs/utils/src/calculations/index.ts index 3fa4e29..f45845b 100644 --- a/libs/utils/src/calculations/index.ts +++ b/libs/utils/src/calculations/index.ts @@ -37,25 +37,25 @@ export type { HasClose, HasOHLC, HasVolume, - HasTimestamp + HasTimestamp, } from '@stock-bot/types'; // Export working calculation functions export * from './basic-calculations'; // Export working technical indicators (building one by one) -export { - sma, - ema, - rsi, - macd, - bollingerBands, - atr, - obv, - stochastic, - williamsR, - cci, - mfi, +export { + sma, + ema, + rsi, + macd, + bollingerBands, + atr, + obv, + stochastic, + williamsR, + cci, + mfi, vwma, momentum, roc, @@ -80,7 +80,7 @@ export { balanceOfPower, trix, massIndex, - coppockCurve + coppockCurve, } from './technical-indicators'; export * from './risk-metrics'; export * from './performance-metrics'; diff --git a/libs/utils/src/calculations/performance-metrics.ts b/libs/utils/src/calculations/performance-metrics.ts index 140a043..f2fe6e3 100644 --- a/libs/utils/src/calculations/performance-metrics.ts +++ b/libs/utils/src/calculations/performance-metrics.ts @@ -1,3 +1,5 @@ +import { ulcerIndex } from './risk-metrics'; + /** * Performance Metrics and Analysis * Comprehensive performance measurement tools for trading strategies and portfolios @@ -18,7 +20,6 @@ export interface PortfolioMetrics { alpha: number; volatility: number; } -import { ulcerIndex } from './risk-metrics'; export interface TradePerformance { totalTrades: number; @@ -156,8 +157,10 @@ export function analyzeDrawdowns( } const first = equityCurve[0]; - if (!first) {return { maxDrawdown: 0, maxDrawdownDuration: 0, averageDrawdown: 0, drawdownPeriods: [] };} - + if (!first) { + return { maxDrawdown: 0, maxDrawdownDuration: 0, averageDrawdown: 0, drawdownPeriods: [] }; + } + let peak = first.value; let peakDate = first.date; let maxDrawdown = 0; @@ -175,18 +178,21 @@ export function analyzeDrawdowns( for (let i = 1; i < equityCurve.length; i++) { const current = equityCurve[i]; - if (!current) {continue;} + if (!current) { + continue; + } if (current.value > peak) { // New peak - end any current drawdown if (currentDrawdownStart) { const prev = equityCurve[i - 1]; - if (!prev) {continue;} - + if (!prev) { + continue; + } + const drawdownMagnitude = (peak - prev.value) / peak; const duration = Math.floor( - (prev.date.getTime() - currentDrawdownStart.getTime()) / - (1000 * 60 * 60 * 24) + (prev.date.getTime() - currentDrawdownStart.getTime()) / (1000 * 60 * 60 * 24) ); drawdownPeriods.push({ @@ -217,8 +223,10 @@ export function analyzeDrawdowns( // Handle ongoing drawdown if (currentDrawdownStart) { const lastPoint = equityCurve[equityCurve.length - 1]; - if (!lastPoint) {return { maxDrawdown, maxDrawdownDuration, averageDrawdown: 0, drawdownPeriods };} - + if (!lastPoint) { + return { maxDrawdown, maxDrawdownDuration, averageDrawdown: 0, drawdownPeriods }; + } + const drawdownMagnitude = (peak - lastPoint.value) / peak; const duration = Math.floor( (lastPoint.date.getTime() - currentDrawdownStart.getTime()) / (1000 * 60 * 60 * 24) @@ -378,8 +386,10 @@ export function strategyPerformanceAttribution( for (let i = 0; i < sectorWeights.length; i++) { const portfolioWeight = sectorWeights[i]; const sectorReturn = sectorReturns[i]; - if (portfolioWeight === undefined || sectorReturn === undefined) {continue;} - + if (portfolioWeight === undefined || sectorReturn === undefined) { + continue; + } + const benchmarkWeight = 1 / sectorWeights.length; // Assuming equal benchmark weights // Allocation effect: (portfolio weight - benchmark weight) * (benchmark sector return - benchmark return) @@ -483,16 +493,31 @@ export function calculateStrategyMetrics( for (let i = 1; i < equityCurve.length; i++) { const current = equityCurve[i]; const previous = equityCurve[i - 1]; - if (!current || !previous) {continue;} - + if (!current || !previous) { + continue; + } + const ret = (current.value - previous.value) / previous.value; returns.push(ret); } const lastPoint = equityCurve[equityCurve.length - 1]; const firstPoint = equityCurve[0]; - if (!lastPoint || !firstPoint) {return { totalValue: 0, totalReturn: 0, totalReturnPercent: 0, dailyReturn: 0, dailyReturnPercent: 0, maxDrawdown: 0, sharpeRatio: 0, beta: 0, alpha: 0, volatility: 0 };} - + if (!lastPoint || !firstPoint) { + return { + totalValue: 0, + totalReturn: 0, + totalReturnPercent: 0, + dailyReturn: 0, + dailyReturnPercent: 0, + maxDrawdown: 0, + sharpeRatio: 0, + beta: 0, + alpha: 0, + volatility: 0, + }; + } + const totalValue = lastPoint.value; const totalReturn = totalValue - firstPoint.value; const totalReturnPercent = (totalReturn / firstPoint.value) * 100; @@ -562,12 +587,10 @@ export function informationRatio(portfolioReturns: number[], benchmarkReturns: n throw new Error('Portfolio and benchmark returns must have the same length.'); } - const excessReturns = portfolioReturns.map( - (portfolioReturn, index) => { - const benchmark = benchmarkReturns[index]; - return benchmark !== undefined ? portfolioReturn - benchmark : 0; - } - ); + const excessReturns = portfolioReturns.map((portfolioReturn, index) => { + const benchmark = benchmarkReturns[index]; + return benchmark !== undefined ? portfolioReturn - benchmark : 0; + }); const trackingError = calculateVolatility(excessReturns); const avgExcessReturn = excessReturns.reduce((sum, ret) => sum + ret, 0) / excessReturns.length; @@ -602,8 +625,10 @@ export function captureRatio( for (let i = 0; i < portfolioReturns.length; i++) { const benchmarkReturn = benchmarkReturns[i]; const portfolioReturn = portfolioReturns[i]; - if (benchmarkReturn === undefined || portfolioReturn === undefined) {continue;} - + if (benchmarkReturn === undefined || portfolioReturn === undefined) { + continue; + } + if (benchmarkReturn > 0) { upCapture += portfolioReturn; upMarketPeriods++; @@ -733,17 +758,21 @@ export function timeWeightedRateOfReturn( if (cashFlows.length < 2) { return 0; } - + const first = cashFlows[0]; - if (!first) {return 0;} - + if (!first) { + return 0; + } + let totalReturn = 1; let previousValue = first.value; for (let i = 1; i < cashFlows.length; i++) { const current = cashFlows[i]; - if (!current) {continue;} - + if (!current) { + continue; + } + const periodReturn = (current.value - previousValue - current.amount) / (previousValue + current.amount); totalReturn *= 1 + periodReturn; @@ -762,10 +791,12 @@ export function moneyWeightedRateOfReturn( if (cashFlows.length === 0) { return 0; } - + const first = cashFlows[0]; - if (!first) {return 0;} - + if (!first) { + return 0; + } + // Approximate MWRR using Internal Rate of Return (IRR) // This requires a numerical method or library for accurate IRR calculation // This is a simplified example and may not be accurate for all cases @@ -826,8 +857,10 @@ function calculateBeta(portfolioReturns: number[], marketReturns: number[]): num for (let i = 0; i < portfolioReturns.length; i++) { const portfolioReturn = portfolioReturns[i]; const marketReturn = marketReturns[i]; - if (portfolioReturn === undefined || marketReturn === undefined) {continue;} - + if (portfolioReturn === undefined || marketReturn === undefined) { + continue; + } + const portfolioDiff = portfolioReturn - portfolioMean; const marketDiff = marketReturn - marketMean; diff --git a/libs/utils/src/calculations/risk-metrics.ts b/libs/utils/src/calculations/risk-metrics.ts index 80cd20d..97daf82 100644 --- a/libs/utils/src/calculations/risk-metrics.ts +++ b/libs/utils/src/calculations/risk-metrics.ts @@ -71,14 +71,18 @@ export function maxDrawdown(equityCurve: number[]): number { let maxDD = 0; const first = equityCurve[0]; - if (first === undefined) {return 0;} - + if (first === undefined) { + return 0; + } + let peak = first; for (let i = 1; i < equityCurve.length; i++) { const current = equityCurve[i]; - if (current === undefined) {continue;} - + if (current === undefined) { + continue; + } + if (current > peak) { peak = current; } else { @@ -150,8 +154,10 @@ export function beta(portfolioReturns: number[], marketReturns: number[]): numbe for (let i = 0; i < n; i++) { const portfolioReturn = portfolioReturns[i]; const marketReturn = marketReturns[i]; - if (portfolioReturn === undefined || marketReturn === undefined) {continue;} - + if (portfolioReturn === undefined || marketReturn === undefined) { + continue; + } + const portfolioDiff = portfolioReturn - portfolioMean; const marketDiff = marketReturn - marketMean; @@ -187,12 +193,13 @@ export function treynorRatio( riskFreeRate: number = 0 ): number { const portfolioBeta = beta(portfolioReturns, marketReturns); - + if (portfolioBeta === 0) { return 0; } - - const portfolioMean = portfolioReturns.reduce((sum, ret) => sum + ret, 0) / portfolioReturns.length; + + const portfolioMean = + portfolioReturns.reduce((sum, ret) => sum + ret, 0) / portfolioReturns.length; return (portfolioMean - riskFreeRate) / portfolioBeta; } @@ -412,7 +419,9 @@ export function riskContribution( for (let i = 0; i < n; i++) { let marginalContribution = 0; const row = covarianceMatrix[i]; - if (!row) {continue;} + if (!row) { + continue; + } for (let j = 0; j < n; j++) { const weight = weights[j]; @@ -442,8 +451,10 @@ export function ulcerIndex(equityCurve: Array<{ value: number; date: Date }>): n let sumSquaredDrawdown = 0; const first = equityCurve[0]; - if (!first) {return 0;} - + if (!first) { + return 0; + } + let peak = first.value; for (const point of equityCurve) { diff --git a/libs/utils/src/calculations/technical-indicators.ts b/libs/utils/src/calculations/technical-indicators.ts index d45fec9..7f78776 100644 --- a/libs/utils/src/calculations/technical-indicators.ts +++ b/libs/utils/src/calculations/technical-indicators.ts @@ -540,7 +540,9 @@ export function adx( for (let i = 1; i < ohlcv.length; i++) { const current = ohlcv[i]; const previous = ohlcv[i - 1]; - if (!current || !previous) {continue;} + if (!current || !previous) { + continue; + } // True Range const tr = Math.max( @@ -575,8 +577,10 @@ export function adx( const atr = atrValues[i]; const plusDMSmoothed = smoothedPlusDM[i]; const minusDMSmoothed = smoothedMinusDM[i]; - if (atr === undefined || plusDMSmoothed === undefined || minusDMSmoothed === undefined) {continue;} - + if (atr === undefined || plusDMSmoothed === undefined || minusDMSmoothed === undefined) { + continue; + } + const diPlus = atr > 0 ? (plusDMSmoothed / atr) * 100 : 0; const diMinus = atr > 0 ? (minusDMSmoothed / atr) * 100 : 0; @@ -602,17 +606,15 @@ export function adx( /** * Parabolic SAR */ -export function parabolicSAR( - ohlcv: OHLCV[], - step: number = 0.02, - maxStep: number = 0.2 -): number[] { +export function parabolicSAR(ohlcv: OHLCV[], step: number = 0.02, maxStep: number = 0.2): number[] { if (ohlcv.length < 2) { return []; } const first = ohlcv[0]; - if (!first) {return [];} + if (!first) { + return []; + } const result: number[] = []; let trend = 1; // 1 for uptrend, -1 for downtrend @@ -625,7 +627,9 @@ export function parabolicSAR( for (let i = 1; i < ohlcv.length; i++) { const curr = ohlcv[i]; const prev = ohlcv[i - 1]; - if (!curr || !prev) {continue;} + if (!curr || !prev) { + continue; + } // Calculate new SAR sar = sar + acceleration * (extremePoint - sar); @@ -834,32 +838,37 @@ export function ultimateOscillator( // Calculate BP and TR for (let i = 0; i < ohlcv.length; i++) { const current = ohlcv[i]!; - + if (i === 0) { bp.push(current.close - current.low); tr.push(current.high - current.low); } else { const previous = ohlcv[i - 1]!; bp.push(current.close - Math.min(current.low, previous.close)); - tr.push(Math.max( - current.high - current.low, - Math.abs(current.high - previous.close), - Math.abs(current.low - previous.close) - )); + tr.push( + Math.max( + current.high - current.low, + Math.abs(current.high - previous.close), + Math.abs(current.low - previous.close) + ) + ); } } const result: number[] = []; for (let i = Math.max(period1, period2, period3) - 1; i < ohlcv.length; i++) { - const avg1 = bp.slice(i - period1 + 1, i + 1).reduce((a, b) => a + b, 0) / - tr.slice(i - period1 + 1, i + 1).reduce((a, b) => a + b, 0); - const avg2 = bp.slice(i - period2 + 1, i + 1).reduce((a, b) => a + b, 0) / - tr.slice(i - period2 + 1, i + 1).reduce((a, b) => a + b, 0); - const avg3 = bp.slice(i - period3 + 1, i + 1).reduce((a, b) => a + b, 0) / - tr.slice(i - period3 + 1, i + 1).reduce((a, b) => a + b, 0); + const avg1 = + bp.slice(i - period1 + 1, i + 1).reduce((a, b) => a + b, 0) / + tr.slice(i - period1 + 1, i + 1).reduce((a, b) => a + b, 0); + const avg2 = + bp.slice(i - period2 + 1, i + 1).reduce((a, b) => a + b, 0) / + tr.slice(i - period2 + 1, i + 1).reduce((a, b) => a + b, 0); + const avg3 = + bp.slice(i - period3 + 1, i + 1).reduce((a, b) => a + b, 0) / + tr.slice(i - period3 + 1, i + 1).reduce((a, b) => a + b, 0); - const uo = 100 * ((4 * avg1) + (2 * avg2) + avg3) / (4 + 2 + 1); + const uo = (100 * (4 * avg1 + 2 * avg2 + avg3)) / (4 + 2 + 1); result.push(uo); } @@ -880,7 +889,7 @@ export function easeOfMovement(ohlcv: OHLCV[], period: number = 14): number[] { const current = ohlcv[i]!; const previous = ohlcv[i - 1]!; - const distance = ((current.high + current.low) / 2) - ((previous.high + previous.low) / 2); + const distance = (current.high + current.low) / 2 - (previous.high + previous.low) / 2; const boxHeight = current.high - current.low; const volume = current.volume; @@ -1028,7 +1037,14 @@ export function klingerVolumeOscillator( const prevTypicalPrice = (previous.high + previous.low + previous.close) / 3; const trend = typicalPrice > prevTypicalPrice ? 1 : -1; - const vf = current.volume * trend * Math.abs((2 * ((current.close - current.low) - (current.high - current.close))) / (current.high - current.low)) * 100; + const vf = + current.volume * + trend * + Math.abs( + (2 * (current.close - current.low - (current.high - current.close))) / + (current.high - current.low) + ) * + 100; volumeForce.push(vf); } @@ -1137,7 +1153,7 @@ export function stochasticRSI( smoothD: number = 3 ): { k: number[]; d: number[] } { const rsiValues = rsi(prices, rsiPeriod); - + if (rsiValues.length < stochPeriod) { return { k: [], d: [] }; } @@ -1266,17 +1282,17 @@ export function massIndex(ohlcv: OHLCV[], period: number = 25): number[] { // Calculate high-low ranges const ranges = ohlcv.map(candle => candle.high - candle.low); - + // Calculate 9-period EMA of ranges const ema9 = ema(ranges, 9); - + // Calculate 9-period EMA of the EMA (double smoothing) const emaEma9 = ema(ema9, 9); // Calculate ratio const ratios: number[] = []; const minLength = Math.min(ema9.length, emaEma9.length); - + for (let i = 0; i < minLength; i++) { const singleEMA = ema9[i]; const doubleEMA = emaEma9[i]; @@ -1299,9 +1315,9 @@ export function massIndex(ohlcv: OHLCV[], period: number = 25): number[] { * Coppock Curve */ export function coppockCurve( - prices: number[], - shortROC: number = 11, - longROC: number = 14, + prices: number[], + shortROC: number = 11, + longROC: number = 14, wma: number = 10 ): number[] { const roc1 = roc(prices, shortROC); diff --git a/libs/utils/src/fetch.ts b/libs/utils/src/fetch.ts index 1c80867..f446c58 100644 --- a/libs/utils/src/fetch.ts +++ b/libs/utils/src/fetch.ts @@ -1,96 +1,94 @@ -/** - * Enhanced fetch wrapper with proxy support and automatic debug logging - * Drop-in replacement for native fetch with additional features - */ - -export interface BunRequestInit extends RequestInit { - proxy?: string; -} - -export interface FetchOptions extends RequestInit { - logger?: any; - proxy?: string | null; - timeout?: number; -} - -export async function fetch( - input: RequestInfo | URL, - options?: FetchOptions -): Promise { - const logger = options?.logger || console; - const url = typeof input === 'string' ? input : input instanceof URL ? input.href : (input as Request).url; - - // Build request options - const requestOptions: RequestInit = { - method: options?.method || 'GET', - headers: options?.headers || {}, - body: options?.body, - signal: options?.signal, - credentials: options?.credentials, - cache: options?.cache, - redirect: options?.redirect, - referrer: options?.referrer, - referrerPolicy: options?.referrerPolicy, - integrity: options?.integrity, - keepalive: options?.keepalive, - mode: options?.mode, - }; - // Handle proxy for Bun - if (options?.proxy) { - // Bun supports proxy via fetch options - (requestOptions as BunRequestInit).proxy = options.proxy; - } - - // Handle timeout - if (options?.timeout) { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), options.timeout); - requestOptions.signal = controller.signal; - - try { - const response = await performFetch(input, requestOptions, logger, url); - clearTimeout(timeoutId); - return response; - } catch (error) { - clearTimeout(timeoutId); - throw error; - } - } - - return performFetch(input, requestOptions, logger, url); -} - -async function performFetch( - input: RequestInfo | URL, - requestOptions: RequestInit, - logger: any, - url: string -): Promise { - logger.debug('HTTP request', { - method: requestOptions.method, - url, - headers: requestOptions.headers, - proxy: (requestOptions as BunRequestInit).proxy || null - }); - - try { - const response = await globalThis.fetch(input, requestOptions); - - logger.debug('HTTP response', { - url, - status: response.status, - statusText: response.statusText, - ok: response.ok, - headers: Object.fromEntries(response.headers.entries()) - }); - - return response; - } catch (error) { - logger.debug('HTTP error', { - url, - error: error instanceof Error ? error.message : String(error), - name: error instanceof Error ? error.name : 'Unknown' - }); - throw error; - } -} \ No newline at end of file +/** + * Enhanced fetch wrapper with proxy support and automatic debug logging + * Drop-in replacement for native fetch with additional features + */ + +export interface BunRequestInit extends RequestInit { + proxy?: string; +} + +export interface FetchOptions extends RequestInit { + logger?: any; + proxy?: string | null; + timeout?: number; +} + +export async function fetch(input: RequestInfo | URL, options?: FetchOptions): Promise { + const logger = options?.logger || console; + const url = + typeof input === 'string' ? input : input instanceof URL ? input.href : (input as Request).url; + + // Build request options + const requestOptions: RequestInit = { + method: options?.method || 'GET', + headers: options?.headers || {}, + body: options?.body, + signal: options?.signal, + credentials: options?.credentials, + cache: options?.cache, + redirect: options?.redirect, + referrer: options?.referrer, + referrerPolicy: options?.referrerPolicy, + integrity: options?.integrity, + keepalive: options?.keepalive, + mode: options?.mode, + }; + // Handle proxy for Bun + if (options?.proxy) { + // Bun supports proxy via fetch options + (requestOptions as BunRequestInit).proxy = options.proxy; + } + + // Handle timeout + if (options?.timeout) { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), options.timeout); + requestOptions.signal = controller.signal; + + try { + const response = await performFetch(input, requestOptions, logger, url); + clearTimeout(timeoutId); + return response; + } catch (error) { + clearTimeout(timeoutId); + throw error; + } + } + + return performFetch(input, requestOptions, logger, url); +} + +async function performFetch( + input: RequestInfo | URL, + requestOptions: RequestInit, + logger: any, + url: string +): Promise { + logger.debug('HTTP request', { + method: requestOptions.method, + url, + headers: requestOptions.headers, + proxy: (requestOptions as BunRequestInit).proxy || null, + }); + + try { + const response = await globalThis.fetch(input, requestOptions); + + logger.debug('HTTP response', { + url, + status: response.status, + statusText: response.statusText, + ok: response.ok, + headers: Object.fromEntries(response.headers.entries()), + }); + + return response; + } catch (error) { + logger.debug('HTTP error', { + url, + error: error instanceof Error ? error.message : String(error), + name: error instanceof Error ? error.name : 'Unknown', + }); + throw error; + } +} diff --git a/libs/utils/src/generic-functions.ts b/libs/utils/src/generic-functions.ts index 0fdd76b..3e4a25d 100644 --- a/libs/utils/src/generic-functions.ts +++ b/libs/utils/src/generic-functions.ts @@ -3,7 +3,7 @@ * These functions demonstrate how to use generic types with OHLCV data */ -import type { OHLCV, HasClose, HasOHLC, HasVolume } from '@stock-bot/types'; +import type { HasClose, HasOHLC, HasVolume, OHLCV } from '@stock-bot/types'; /** * Extract close prices from any data structure that has a close field @@ -16,7 +16,9 @@ export function extractCloses(data: T[]): number[] { /** * Extract OHLC prices from any data structure that has OHLC fields */ -export function extractOHLC(data: T[]): { +export function extractOHLC( + data: T[] +): { opens: number[]; highs: number[]; lows: number[]; @@ -43,12 +45,12 @@ export function extractVolumes(data: T[]): number[] { export function calculateSMA(data: T[], period: number): number[] { const closes = extractCloses(data); const result: number[] = []; - + for (let i = period - 1; i < closes.length; i++) { const sum = closes.slice(i - period + 1, i + 1).reduce((a, b) => a + b, 0); result.push(sum / period); } - + return result; } @@ -64,7 +66,7 @@ export function calculateTypicalPrice(data: T[]): number[] { */ export function calculateTrueRange(data: T[]): number[] { const result: number[] = []; - + for (let i = 0; i < data.length; i++) { if (i === 0) { result.push(data[i]!.high - data[i]!.low); @@ -79,7 +81,7 @@ export function calculateTrueRange(data: T[]): number[] { result.push(tr); } } - + return result; } @@ -89,7 +91,7 @@ export function calculateTrueRange(data: T[]): number[] { export function calculateReturns(data: T[]): number[] { const closes = extractCloses(data); const returns: number[] = []; - + for (let i = 1; i < closes.length; i++) { const current = closes[i]!; const previous = closes[i - 1]!; @@ -99,7 +101,7 @@ export function calculateReturns(data: T[]): number[] { returns.push(0); } } - + return returns; } @@ -109,7 +111,7 @@ export function calculateReturns(data: T[]): number[] { export function calculateLogReturns(data: T[]): number[] { const closes = extractCloses(data); const logReturns: number[] = []; - + for (let i = 1; i < closes.length; i++) { const current = closes[i]!; const previous = closes[i - 1]!; @@ -119,7 +121,7 @@ export function calculateLogReturns(data: T[]): number[] { logReturns.push(0); } } - + return logReturns; } @@ -130,19 +132,19 @@ export function calculateVWAP(data: T[]): number[ const result: number[] = []; let cumulativeVolumePrice = 0; let cumulativeVolume = 0; - + for (const item of data) { const typicalPrice = (item.high + item.low + item.close) / 3; cumulativeVolumePrice += typicalPrice * item.volume; cumulativeVolume += item.volume; - + if (cumulativeVolume > 0) { result.push(cumulativeVolumePrice / cumulativeVolume); } else { result.push(typicalPrice); } } - + return result; } @@ -156,11 +158,7 @@ export function filterBySymbol(data: OHLCV[], symbol: string): OHLCV[] { /** * Filter OHLCV data by time range */ -export function filterByTimeRange( - data: OHLCV[], - startTime: number, - endTime: number -): OHLCV[] { +export function filterByTimeRange(data: OHLCV[], startTime: number, endTime: number): OHLCV[] { return data.filter(item => item.timestamp >= startTime && item.timestamp <= endTime); } @@ -169,14 +167,14 @@ export function filterByTimeRange( */ export function groupBySymbol(data: OHLCV[]): Record { const grouped: Record = {}; - + for (const item of data) { if (!grouped[item.symbol]) { grouped[item.symbol] = []; } grouped[item.symbol]!.push(item); } - + return grouped; } @@ -186,6 +184,6 @@ export function groupBySymbol(data: OHLCV[]): Record { export function convertTimestamps(data: OHLCV[]): Array { return data.map(item => ({ ...item, - date: new Date(item.timestamp) + date: new Date(item.timestamp), })); -} \ No newline at end of file +} diff --git a/libs/utils/src/user-agent.ts b/libs/utils/src/user-agent.ts index 49eea93..ac76234 100644 --- a/libs/utils/src/user-agent.ts +++ b/libs/utils/src/user-agent.ts @@ -1,30 +1,30 @@ -/** - * User Agent utility for generating random user agents - */ - -// Simple list of common user agents to avoid external dependency -const USER_AGENTS = [ - // Chrome on Windows - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36', - // Chrome on Mac - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36', - // Firefox on Windows - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:119.0) Gecko/20100101 Firefox/119.0', - // Firefox on Mac - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15) Gecko/20100101 Firefox/120.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15) Gecko/20100101 Firefox/119.0', - // Safari on Mac - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15', - // Edge on Windows - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0', -]; - -export function getRandomUserAgent(): string { - const index = Math.floor(Math.random() * USER_AGENTS.length); - return USER_AGENTS[index]!; -} \ No newline at end of file +/** + * User Agent utility for generating random user agents + */ + +// Simple list of common user agents to avoid external dependency +const USER_AGENTS = [ + // Chrome on Windows + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36', + // Chrome on Mac + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36', + // Firefox on Windows + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:119.0) Gecko/20100101 Firefox/119.0', + // Firefox on Mac + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15) Gecko/20100101 Firefox/120.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15) Gecko/20100101 Firefox/119.0', + // Safari on Mac + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15', + // Edge on Windows + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0', +]; + +export function getRandomUserAgent(): string { + const index = Math.floor(Math.random() * USER_AGENTS.length); + return USER_AGENTS[index]!; +} diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh deleted file mode 100755 index 25fefb6..0000000 --- a/scripts/setup-mcp.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash - -# Setup MCP Servers for Stock Bot -# This script helps set up Model Context Protocol servers for PostgreSQL and MongoDB - -set -e - -echo "🚀 Setting up MCP servers for Stock Bot..." - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -# Check if infrastructure is running -echo -e "\n${YELLOW}📊 Checking infrastructure status...${NC}" - -# Check PostgreSQL -if nc -z localhost 5432; then - echo -e "${GREEN}✅ PostgreSQL is running on port 5432${NC}" - PG_RUNNING=true -else - echo -e "${RED}❌ PostgreSQL is not running on port 5432${NC}" - PG_RUNNING=false -fi - -# Check MongoDB -if nc -z localhost 27017; then - echo -e "${GREEN}✅ MongoDB is running on port 27017${NC}" - MONGO_RUNNING=true -else - echo -e "${RED}❌ MongoDB is not running on port 27017${NC}" - MONGO_RUNNING=false -fi - -# Start infrastructure if needed -if [ "$PG_RUNNING" = false ] || [ "$MONGO_RUNNING" = false ]; then - echo -e "\n${YELLOW}🔧 Starting required infrastructure...${NC}" - bun run infra:up - echo -e "${GREEN}✅ Infrastructure started${NC}" - - # Wait a moment for services to be ready - echo -e "${YELLOW}⏳ Waiting for services to be ready...${NC}" - sleep 5 -fi - -echo -e "\n${YELLOW}🔧 Testing MCP server connections...${NC}" - -# Get project paths -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" - -# Test PostgreSQL MCP server -echo -e "\n${YELLOW}Testing PostgreSQL MCP server...${NC}" -if npm list @modelcontextprotocol/server-postgres --prefix "$PROJECT_ROOT" >/dev/null 2>&1; then - echo -e "${GREEN}✅ PostgreSQL MCP server package is installed${NC}" - echo -e "${YELLOW} Package: @modelcontextprotocol/server-postgres v0.6.2${NC}" -else - echo -e "${RED}❌ PostgreSQL MCP server package not found${NC}" -fi - -# Test MongoDB MCP server -echo -e "\n${YELLOW}Testing MongoDB MCP server...${NC}" -if npm list mongodb-mcp-server --prefix "$PROJECT_ROOT" >/dev/null 2>&1; then - echo -e "${GREEN}✅ MongoDB MCP server package is installed${NC}" - echo -e "${YELLOW} Package: mongodb-mcp-server v0.1.1 (official MongoDB team)${NC}" -else - echo -e "${RED}❌ MongoDB MCP server package not found${NC}" -fi - -echo -e "\n${GREEN}🎉 MCP setup complete!${NC}" -echo -e "\n${YELLOW}📋 Configuration saved to: .vscode/mcp.json${NC}" -echo -e "\n${YELLOW}🔗 Connection details:${NC}" -echo -e " PostgreSQL: postgresql://trading_user:trading_pass_dev@localhost:5432/trading_bot" -echo -e " MongoDB: mongodb://trading_admin:trading_mongo_dev@localhost:27017/stock?authSource=admin" - -echo -e "\n${YELLOW}📖 Usage:${NC}" -echo -e " - The MCP servers are configured in .vscode/mcp.json" -echo -e " - Claude Code will automatically use these servers when they're available" -echo -e " - Make sure your infrastructure is running with: bun run infra:up" - -echo -e "\n${GREEN}✨ Ready to use MCP with PostgreSQL and MongoDB!${NC}" \ No newline at end of file diff --git a/tsconfig.app.json b/tsconfig.app.json index 645a4d4..009565a 100644 --- a/tsconfig.app.json +++ b/tsconfig.app.json @@ -1,19 +1,18 @@ -{ - "$schema": "https://json.schemastore.org/tsconfig", - "extends": "./tsconfig.json", - "compilerOptions": { - // Override root settings for application builds - "composite": true, - "incremental": true, - "types": ["bun-types"], - // Modern TC39 Stage 3 decorators (TypeScript 5+ default) - "experimentalDecorators": false, - "emitDecoratorMetadata": true, - // Suppress decorator-related type checking issues due to Bun's hybrid implementation - "skipLibCheck": true, - "suppressImplicitAnyIndexErrors": true - - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist", "**/*.test.ts", "**/*.spec.ts"] -} \ No newline at end of file +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "./tsconfig.json", + "compilerOptions": { + // Override root settings for application builds + "composite": true, + "incremental": true, + "types": ["bun-types"], + // Modern TC39 Stage 3 decorators (TypeScript 5+ default) + "experimentalDecorators": false, + "emitDecoratorMetadata": true, + // Suppress decorator-related type checking issues due to Bun's hybrid implementation + "skipLibCheck": true, + "suppressImplicitAnyIndexErrors": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "**/*.test.ts", "**/*.spec.ts"] +} diff --git a/tsconfig.lib.json b/tsconfig.lib.json index 1d94681..4224ec1 100644 --- a/tsconfig.lib.json +++ b/tsconfig.lib.json @@ -1,17 +1,17 @@ -{ - "$schema": "https://json.schemastore.org/tsconfig", - "extends": "./tsconfig.json", - "compilerOptions": { - // Override root settings for library builds - "composite": true, - "declaration": true, - "declarationMap": true, - "incremental": true, - "noEmit": false, - "outDir": "./dist", - "rootDir": "./src", - "types": ["bun-types"] - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "./dist", "**/*.test.ts", "**/*.spec.ts"] -} +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "./tsconfig.json", + "compilerOptions": { + // Override root settings for library builds + "composite": true, + "declaration": true, + "declarationMap": true, + "incremental": true, + "noEmit": false, + "outDir": "./dist", + "rootDir": "./src", + "types": ["bun-types"] + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "./dist", "**/*.test.ts", "**/*.spec.ts"] +}