From 60ada5f6a30a43b90623809a34f4f1156458be3f Mon Sep 17 00:00:00 2001 From: Boki Date: Sun, 22 Jun 2025 18:14:34 -0400 Subject: [PATCH] di-refactor coming along --- .../document_symbols_cache_v20-05-25.pkl | Bin 102226 -> 180863 bytes apps/data-ingestion/src/index.ts | 10 +- apps/data-pipeline/src/clients.ts | 33 +-- apps/data-pipeline/src/container-setup.ts | 34 +++ .../handlers/exchanges/exchanges.handler.ts | 22 +- .../clear-postgresql-data.operations.ts | 11 +- .../enhanced-sync-status.operations.ts | 9 +- .../operations/exchange-stats.operations.ts | 9 +- .../provider-mapping-stats.operations.ts | 9 +- .../sync-all-exchanges.operations.ts | 55 ++-- apps/data-pipeline/src/handlers/index.ts | 33 +++ .../src/handlers/symbols/symbols.handler.ts | 21 +- apps/data-pipeline/src/index.ts | 244 +++++++++--------- apps/data-pipeline/src/migration-helper.ts | 37 +++ .../data-pipeline/src/routes/create-routes.ts | 26 ++ apps/web-api/src/clients.ts | 33 +-- apps/web-api/src/container-setup.ts | 34 +++ apps/web-api/src/index.ts | 243 +++++++++-------- apps/web-api/src/migration-helper.ts | 30 +++ apps/web-api/src/routes/create-routes.ts | 24 ++ 20 files changed, 582 insertions(+), 335 deletions(-) create mode 100644 apps/data-pipeline/src/container-setup.ts create mode 100644 apps/data-pipeline/src/handlers/index.ts create mode 100644 apps/data-pipeline/src/migration-helper.ts create mode 100644 apps/data-pipeline/src/routes/create-routes.ts create mode 100644 apps/web-api/src/container-setup.ts create mode 100644 apps/web-api/src/migration-helper.ts create mode 100644 apps/web-api/src/routes/create-routes.ts diff --git a/.serena/cache/typescript/document_symbols_cache_v20-05-25.pkl b/.serena/cache/typescript/document_symbols_cache_v20-05-25.pkl index 44c37799bcb459ebc02e092679ca96386b449943..937683cf16efbb6ed17252c71c34c82e78fa9e37 100644 GIT binary patch literal 180863 zcmdUY378yJwSOS9Wo9zj7Xl#(TSCZWmSi%Mun5^8bS4Q&5D^3PQZwC^p6Q{Nkhmct zD9}D1kHH1yA?i~B_YH7=i2ME#?WrsDQTaeMebr|#+MuA6&qS5Nv@mwbGuGpGAh z-TIy7o_p?6uV}jC_-V&Y!~b43xvDsjPQ`LP@l4k0Nk+#LX)BxS$z|g`2@819dr6-HiCW}o6Q{#!rNN2G*8yz3DCQ+g}myc%i&Oe*d zsc~yEvbQ)Ro{VOr@w}Ctj7%#Or&}NezfE>UUX+{_Ij&GBcIB+J70;(K<9kb2N_L8$ z?1i7qE+l7{|5^+JZSiC(oyc0_(9&XCIunnguyfn#g=}iFc*#gAZEfo5=}BhBte#lr zV5%o;O=JL{d?tRdJC@1!)Ck^W=`oWX#m;Cfmq{1$)}ClSIa$28o_FcYT4{KiLv-z$ zn^`EfOhf@0Qi9N6d)W?3dMlnVR!HYl>Dd07U-d7_;kPw2f&E9W^aNgq zx)82d4l?-HZ0QM*S8hS+7s8=#F{|P`OsJSbZ}nd3dGO%nUJ3t_d!?g)QSOxtLvxQ& zXtJ4c$l+mV;-(#5tJw}(~qr=GHz= zPp)?ANy>%|pLrZu7y530^4XG3cODZAJr#F{N+{?VuDrCkZEDYb)YX<1S8HI9f9{}C zSzz^GB*%l2;uCV7q3r(4|y%aCrFv~7LP)`2c*u%}Q^TXLI z;erJmC<$X}jb$^}=3t`2W5KOzLz$ztU}DeX428b$knD@tb1MwRAduaLfozUPp`PFi z#x6CC#D?LPIuOwJGK|EAvAyeW24UQ27{)dS#;sP`i)4%1Apcp7nk&4MvT~9$K0%h zxPP0}EMl{14}CB9*A-%Je*99j6{=j+d)Q~c+OQ$c$fw3ESOqgS;Y-Pu8ckxu6pC8O zdNE9zf?PH%5SDkF!4gV@cFv?nz1&Q@-9cFs zE=uxc05M?za>OJyOrWcssk)f!Oh$oxpBW%=-oo6FykMr$uJ&6kr%RU%fEfX;5e6U^ zm4*eI9RZP;+@%q_90AHb`Ml^MSO$rUBHcNOoG%kyuQpL)U&)!MUJMpGQH*k7JHeg5 z%dH)YPOLg}c|4j<$D;9r-ez{Hhmv%^)lw%<>;qMrsSy*NR0guNFvw5Ah3>vX-KZB< zX%r*2+r(&cT%Kw?=X|VxM27nHn)SEqTVI`u}c5 z{l?Aqk7nBKnyqJHl_%^nGfUmVT;-W1VzV<#dNI$;lF=~Gb$oqimX4Zyp-n8_1cfP% z^sDnIVlU%-N-s9kb$!zzb%|jtm1z}YmYdaPN$d?AlzOqs2^h1z(;1C`lCjDiYO5qR zmIRe0JG~ezbuU*SjCy##VJIsW6?)<&gXTIln#9IonG((FVs5`_6wUL(h2}+SG>Ofk zSzRm`G%q%c=1h0MJPbv8Zy&@2&LMT+B=#_8pL#La%_XOwOWdaA){elIyjlnIbVGSm zPVAjk+Nlw9HNPK~W2$0*l^M+&&p)4UrrmD-*4syj^&< zfFB22C>%_}GeYc>88}Pd%cE@)s~k2PoE?Gl8tS?lQyw$7C!@K2R`yUTE@7!Q&d4Kb zEWaq&>;wskNtRi7pBWvS zT}gY7nMS(?(3H&Qr4 z!QR1{BqK)b@<6`aD0kDUpGZizUm!g*vDu;*eZQC&a})W`nsk(}8n(HXbY^ta=i+U` zWStt5?VN)3VjDp$7%0DI8cI5^p$4m5p@y=r3Y3i4%D|v}$_$h^oA`i9Z+h5FyWI#r z9dclAe#Qjw`_!@We!*s4i4i0A3|IeCjFoiz1(H<~n~jz9y&Nkm#9Rmetx4_v)v$JJ ziIw#5y=o}8a1O2)!+C{ZplqFPHV0SEwJ;@Z!bA>w#9mtkN=D2RD~(#^JTp)lkCneS z>`hH@w4A?S`5tw!{IFoNOB5L~VxySwr6~r>_myt%-StUqOs17HE%g0NDKJ!sxgNe8 zEyvUd>?|{8>2`%pu}tEKw=ZP;`vYnqiG3Oeq+YDNW*>42qd*QC2C_A2MH6sLr7tW$ zsK%1myErWMV&$?x*;QtoQy4!up1F4Xyjt3|QL=ivFFTz=-)qENXSu+nv+Oji{Jj72GSRpffTz%qwc=NOuOB|-8~f2?$SZ;nZ9Hpiszf`jUH>`egK8j&&E}YT zF;9+Z6v#Ii2C}9c^C2~s#Ab6$y;vP`OuB)bEE1c|F{_KY&T@}QXL*ld9Zx04By8kl zk=SgGsTcF)m_{Le&H?Wy9$e8BJy5K+edF z!|SkMzR%Y}vf%~HYTy!|8?L*8(z#HPShi-y_c^a$^Q_uZ=R(8t7%)qGZnL5*D1%uNEayW;1Lj7%~$@D(+)V&X}*+MaWFZVvg z#!jo05*u{!OAI5~Jdw>D_O}#awi-raWA~xVhUj}4Mq*cv&<(37>2PeYmKI0>!Eb; zxVr0(JgP);xP^KlwvM5eD_u^th^z(PXgHFRiYv@6oK#Z0^1qE*c& zLp>(!K#rZn?%_O3FIKt5)hwPLH;m_r;&5+pD|&%YttVjoMBuw#sto^Ju-T)utBVOg za~0nT{pHcejx!zR6XNb&Qlow**tp(C8BQ57VwaU(?l5aRcKk_kf7up^ji)>+o23^k zr@Q55wd68|th{fT>3Znu@@t8R%^tF#7dr`_#SN5?nG85j8g`Hx4;?31R!WvfZ0>-9 zh+eEWFf^YwgC=%6UpHw5kDFJk+j}x0sjP+ub#VWHv{>==KXaJ9xG=OiK zX*Zr^WHev;f|+)^hRYub!gQVAQ`;}Gw{d-xUaZ`8n#QxM+3aK=k0;Z;J}2;$2g<}| zYctiw-0sz=eJ(MK=kfA90ArRv7BJ3|`c-1HFjf}}2F5cC!#Kx1u#D1pX^_mI`XjZo z6MKMjcD-2nlxNe<-ftLHwu-~R`M4TRVzY4S#meWcn1=JbM#0I1&cCZeC$TT(tWz&m zE{eH*dnT ziwv8h{}(zCVtrD_Lu_`;t{1~l5Dbt>GeBZKf00Q8*k`8Q#`F1$&9vK%Rk}(z(<}Xt zx>tIp6kVCB6C*}!9LNd_cAK`iRHUEMYjNnht0X_8RJ}Y1il*57oNXU<4Np&9G-fy z;P8B-X*^S7V^$&sZ`@4#TaEHtHJ-%o=kU~v1&8N54CBe2jL$^NUkEhi9SOvKK8L1W zYypT#(ehYN9;X}1@sikVa$H@^%}d^AGAcb}M(fzW8;zzPFw<_g6XmRx86rKc_I+Xx zan`CA!)l6a1xC^Ql3_GkCse0PGg$s!jU}-+b6D!dDwphH4LWz`J`W|fh6{N4oJON`M6xuK>o@MkU0K4jF&c5fVja- zyN!=OUu>q`ZvUhkf2!l^@U-KE2}@HT!%f8JmSeDjr3;pGRJVC2mBygXLa`~lp4Gjn zG=uN9wvF*PUxLRv79BpGFp~zOE}z}=R86pqC3seIi$TwtjTc{9+}4xKj9ERg%)wMo z)`DFOJ-K`)ey}^1$@ipFv0P6)leKzik1BkbjrYW&Ijd(g5Srsyo6k)qm&xb$choq3 z8lKKzTH>>qmVO{Bqr?vm(~}HiT3c2+d>n&liNBolq0$dzL`!^6W@^+*SD8T+`>ns3 z?6+FYwA*;U^_<~;YeuPK@Ha8~ebHsfEA@!Ij{bvGn=2KviCwuc(jb;Q%ou>1XU!>LxqHIOMvpH2 zGLY_8L;Auhkdm0xD@2HW0(hgGST(~7NuvHns+WtKua@*hY<7WobuqVuvJaJEYM^qZ zVNJK%nb_8-eB=sKJl!v_l*8r=1e=}2VZ?}C?)1x3;WZsza;dnzH2#PXn|)t_Ud;1C zI;^^w>n}#pyv{J1HI*DMQ)5YNG%+QX)y0Cr@>Vlg zVv=k$X4+=j?e>nUt-DOb+$StRj+n$AG;}`>W1*L6AF8>S= z6hW-qLg~$+(nwC7at^PcmPl#*@JixyNeL@hx?p+V6@;t%(4|0+NkbJMZhU|De)Q|fdZQAiFRn8dsno^%#@=3eJS1M$GCHLd&%v>8?! zCj0GgRG8=fV@9S&YKK#YJxSbTX>oY!h$t_-tk_=#B$b$pTTv|aEY3w;Ho zNx(^xsq|#Bt=M!B-u*rq=_)p7qvNC2B;Z|a&N&A@;Xj+wsc~yE^6BD?cru!e#_=?X z$jm}131N#ZUH}Yxwr}zO=a>cG3ybm*u!{WWA>)AhsFXi9D(d$nt*ia-Ww#AdF zbRuhwPac>o8MQIh{RkQX#vB&at+2la4RZRX@4-KgHhla0*x)w%2k;STwtom82(1U= zW}yz~nLH4GLa?!8dP*Y(gGG-RT@4rV{5OalaD38sTGt#;TTwV(q_Ryd>NedYZVr8s z=i=`WY%D1$+r)?w8~0d?HbF=9G&Ih*x%>%lV!&MNU%`b7WxaPnkAFcrP2mRmdmj*w zsv$m91!6{w*tqaORy!>ruWVA6TI*xt-tazMkJgEe`#Y3cuP)|l{rJ`Zv_2DW5`fk{ zyKel$3z?(}UYu*v!K)Dq7TyyY4R5R3dMAYSLh;oDFR?KTR>E6dELeC?4+!2CIE$~4 zhEq3vSZ-243xj_Tv3(`G7 zg4lTUy3!Z*VmN2!`eGmmUm6008DG3c9qovXDZdio>SDnn95stD`eGoXizc(r;rik{ zF}iT;p5kb^%Gk|};?5EFB9G#B2sS&4W5kH98dBv*LwrPwF&H1s@~}X!BWX!$&%?-+Pvr?m?DbC(w4i>Q8HvELiCBI z$W=39eWwA7gzCyb(W#!mI3OC)E9DeX!EWI!xw@EW zTe-7P%m^~Q!s)_DdokTRw6ryNMv#H`DmA>s#zUBu@ao0N z^D0mX3k2S|0l}*)N;3#wtwxyG?DU0RtnBYWA$+n~gt2${eGrcLe`b3J-2E3w>Gv74 z&*63>9pzrmU;p+JVefMP_AuqM0A6Sq$$Lw>sMm!sgxc14R zU1~etQ^k%MF=Bg`P6QGu_6G!V8-kpV!buLE1v5;LxK|DC$E$$Lh!GppZ8?>9-_BM! zzFdM1?0%N%dOl)>qQ!nO8 z90G|9uMGvpOn`Vm9UzF!M&IgU!P@bg0>XH%W5=4bhB4*))uv2rb_Il9Y)Rm8_}{_; zGh@vUs;!yWi@B&=T`X8@{-9Zy1FO1UZT30bp+;LxIZ=x+61f;b>)ULfPRuLT5hdor3+PA@Vexm(om76qFv zs4-&1F3;cxRj_& zK&I0WV1v4!dRnI%+__c2WyFYGuBW-GuXX=fmuP@o^&$4@T<5D7^Gx8j%m^St%)y(O z+c6thglIGS9Ilzn7sp;$*~?P-WGPhmGUP>SLq4U7Av0pcUI=z3#!cll9<^HX32I9w zHg4Ecmf7@T9!ows2unUQAPbnnQiLUHgo(|LhV){dr3ir_JYW{#z=q7r%|3@4E7}X= zrD)+RLcB>GA>L9YLNH>)4t~I~)hxW0@vQkbM$Zm`ai90-n3i;AbX3lUn2EgUrATr| z(=<5T4F2=Z=+XCUxS;LIE5#~o#q`KzpHU+zVT>W=VLY$}d=gHlni(zRGu^IF!eLyx zv#CA;XH&r$Ja9JERoGZUWbd(`P336?olNE2VremQ3@1|^ec5Hj-fHJk9VK+ssZ{gv zC*nvdD@*m0+@#`Ys!sjURGU3VQ%$QlnrZ^zjofZugC796@@T4Y_!0xF^V`khw{R{> zR>58^9ZltAMoIW#aYiDQg)^@*a4wl`0|Sm{ItU+;Ry$4C@!V#QxXn4zZHSHARr6C5 zI}BoQWY!9M6#ik@uwII{2!&xehbB3CXh~#`>@C0%=aow6QY#@Ml+Yt7f!ODAO3;f9 zfLM@9xY>vj?3>`S;w&l$)@Cc4&A_O-64jY%R9`7jg#{rV4 zzD}UpA)!ia91%s0q(R|xxEB;ibKtf?sNx4Kx6IDb13oLfdBr)Z^Ms4113naUrO%Sf{eTVj{qPf$9a4 zsS zqdCcF8E_8@aOX+j5}S>$)y0AY_w+!(?V=@o4!CkmVBn1lc)KKciH)1Um6292hOh4n z65hT*;Dyy?&btw|A02QRaQ6vtVF{NfFc6z99O}hzzCB2A&kY1z?_qd8f^J%13y0x( zuq8I0yrMK*y%=V}L1Mcj5NunCu|RNbFU>mpti$XS zwkg+Ph>fQVDkH334A+zf3GI=_Ld#g~EdpA(7DMb)IcW7_IJpxfv^RzlS}Lr-JEJ^# z0kLuYyV7c_ivU zy(im?I{5^h$91Y+Z2Zl&Go#exnA_nL&(U3F(@{nMDiA}B1hIRKY0uRl2< zp0%QR>jGHVL?scizDZse#7sTCQCJJSw$0;+9~Nv_!KAhp5|al_#K!DiIAVFd(!5gA zu$9&=rR2gwi+XbG9ik~sQVTyQ*v(uENlZ?7iH(y#ato)!iYi~h>ppR7IOf}f8L@GU zqV%xpVs7yI44S{$MS5Q`>0!9KO{|V8y`IU(K>U6It6V`M_6`nKy_jdj7yzt~1p;fU zfYo2&^WOxna$oyy!N%8KDb1A;BQ{Q*$PT2On`L^bcZs{pV*+C10SC%P>BT&Avp+TO zL%$3}+3awOf%cthXo-zy_$Z;(i+P4y0YLk#Noa8?bpWy8XQrRwW9Ci#-%UP)+fB@o zlLKi{Gm{$JB`jS|4el0fHZ>qIIW-`*r*swS< z)SsM0GHLFa!X)H0cavbVX)cM)kTN}DV=pE5fo;~|iA*-{voh8#?k`&+vGHB1%22Hr z^MvY!XkBJA$rUCI*bNaCXSVprC07ez<@AEsY&BOe<{1PB0Bm<4z)nACNzs<6*{l(m z$^|iEvn#@@i@C#|05IKX5>wOBcCo?69~Et9kEF&E6lxXKCnnI&7Dgfm+S3G^4YVXC z2U=p6m#~!U3C)V-N{r#oQ3nHJ;~E2Hl+}yjY0|FYZb!wKjk1>*)P1>{)Rc=yrFlq3 z(enhTa4MFk2(nzT@kAjds3azDDAT9T(t=4!xdsMNAQT2Rrd$~g;_E}u3^kSab z(JRcWdL$54HzTh8BL6~xs=SJl*lfY8x|r)g0ic=<1l6{YLOShx8f1|GRxZsDdo5?K zdNEJQD*#|$9tf~aIXJf3S9x}^8dPGl6CipqPwp20sJEB|73&`XbXfFqyN`<*da;V_7iM97O8~ICCndl?l26ik3C;N<6HqLGa}pje zWoqYP3c0DLr}lFS=vpVi=rp+o$=b*?IALL0A=z8`R{=Jw4^H{&%Zev4Cmx8~yQdq# zX%OY7)13m*1#MjKoC5J7R0C=H!efJ>$7u~6{oq`8Y4Q9iK6J&kmK9IeJ$?C?4h-N? z?}SGpUC+r`*+Z$g)g8}_=cB1{IFO{u`4}g<=VN@b0-1fsh|E#VAe5YYEG_PwDoX0> zE-UtF5mQPz=z+}^#oN@1b9U?#*um@Q2s*{_7#;fKF*?}eF}@DyMwZy$zz-M{h~L6o>o*l_n9L})#gnOYB5RFL9+-57ho6TZ7F%Q4%(d{ullB+kPdFUo zOYi}QV|Y?WmVtDdxXTQ zoDrp?h~{`So`lU>KGe{qRs*qd#g$SG^u4SGVq@YKs2cvuj2i5x;JV^~3`XPV#AYp)zL$X|cHqE17Ywki zW0~>MOd{sXWaq1qB{mz9>3bPjVh0|Q+dGYfWH}bJO=NQUQCJh=3)%&0Xo-zyrYoZX zeJ?{xY+Mf#Xf#+D47AM?+00?6lKBV*v(>l~8z=LWxYGABuEY+!`#L2UTwAXxSh>8< zV6ae)EV0>OK;O&A5BF)LGbukpgFZzUQSS15n4Z$rge7cP3Nr1;_}|u#^}(7fkZ5^KDK@&*54mX#0Q7^ht{uOw|*#|7{Fn6^6&OsII`6~ z5y@Fd29az*vKz@2NJf$5kh~PhjYw`oau<^KAo&E6N059S$zw=lFev9PKNSdbunS*3G zk~5JEAh`g^#Ymoy#6ofnl2;(P8Oa?;?m==Nl8+(zJd$rCc?!v|ko*ZrYZH)pNR}Zv z3&|joEl74FxdO>3k{pujkh})TZAk7yaxanxkvxLr>qs6$@&uA!BKZrF&SoHsk(`F4 z7s)wDE<$n%k{2QwL2?+$^+;|(@_Hn1L-IZ(4k~sUG-LZQyd%CC$o?aeP4JEi zdp44jkgP?5Zzi_4AlZuq-#2KFBFQ1S4#}-Z-hkxoNZya+(@4I8)%K6Uh&e{5O*Shve@_I$*`2 zy$H#vNP3W*jpRZk`;c6P$^yBRkQuSNdADN z75YsZS0vfXker2N5DCtl*}IY8JfNLGGJ)i!NM3{FP9T;&0_}A$J&VEAshqo&Y!}E4 zvA(`|?}peqYkk6s^$l)VH((9)_4oJpS=NTwz&ZfZehBX1TzgP^ZB@tMc{IYT}e12*?l}|;}sTW&2Ht)%14y6)MQC8H$s~i{I#n!`L z4G$H$+^k^eads+JYCjbAM^dAc_VvJr zy7(*MBhqExKv(nJY>v1YtfKVX>{7wTH9oi2AVyciU7D{0F#)w`o|t^_G^thHq`BhW zu;km*Bx2(R5M`6}Vr_1dAWgmv+E&VPedPQ#8zZEP_FM2C`cnNAt_u@DwM&VTvSTEk zm#tKq2cwI5YF%6`*tq;&sS8Go*r-C;Jm#z%vGVa`H=MBvP7N-<(rdm<`%V-MfthlT zt(-2{sB~rf7%^hws93bmb*>Y|7R-w?G0I0g=YLiL*{L>ay7X#wLve?euAv$+8OSaN z$89ATy(lt4V3~kRjyS}|HSo&DRTmSD1CM(s0FQgbjK|sUgzHwgP^Sa%J}Z}lPRSR% zC#&HlHZD$8!dqP|P;SAud_(K3jI&G$r z6q7nUbhTdv@D$G)b+w=Z#;=V&g=^|=z&J7%%cOlK$t%?XqpwQ9V8n=RB4E&^<$ysA zWdlZaG2xcZ(lKWkRC~l&t&SMPW+O&*F*jnIfZlAlb3V;z#Bi;Y;1xFND}y^r4KJ}- zc&m#A3U6;P@J=N~ULydOQ)FVF!==c2G1N<-0H0$NU`&x`2jGP#8hwhOdc3(tpTdnL zOID_0<*HCO)PK8kwRC>#gyGBC@(#gfvt>q%*tpV17{1%iW6qZ8((9ybnHtJ=e$~Z9 zL(83Cb39WRhkaI+18b(dcD_JqKxz}Qhqsx|EtuGB*|@q`pcee9U;wUt$gx2UFR|I2xVl)N@cwh4@YXQo7*fMaY!=?? zVu8Z@mSEtW%8+A|09YP!5Stxx=*2ukjvzX?cLxJ-Dr6n5h=3?B z#@a8~>;fQ0jM%vSQ3S+tRiY!F&5XmTI^r!kK57%YMHA$eYsAJ1jWSg0#c+?C8>*iO zzzct2)PCLSR7b+f#j~jiTuA9nF&MpHqDGh4+cr+9e)t3GZwn(;(UC(o}k z+5F^+T?HrYwC{x5IP2Y8&@0I&mqD2OBj-%9JSSEq_-~-9U6?Ui}gh}CN>TXTG4fh-oE~k#Gti)WMoj<0@E2yr!&`<*4rw! zzO?F(f;!W)^`)s$oNkRLCebV|g0@G_w0GdgI_4;-?eL}iTNrIztYAYw>^TamV7gjT{3TC?L(*jnS+u~KPhol5A2=N>1poA1SLcmeTYFPjoP zUxLEeTsLU8ve^tAXz}=F15E38;2Q6Z`F%oR$wBwy{5jXxqv|-%u2w) z-LNUuUyra_-6Ig2jd6N0Jf__QYLkXg>%z^#c;-cE@2d}Uo1->2VxPg8n_di$<~9M` zTMYr%VT~V3LFsi2OwLaLj+~%2H)7-ah?P#N7c*sU?=gg0dlL4E?~d74+#hI5)SwX? z4=h)LrWeC~v?fgLK|`R;E$_9b6KJIXr_Mwo=L+!Pg(H>(u%ik$LTPlG@Y&L*Y7c&Uj_f3H2o${*eu++J4pl!3 z_n#(^Yb*8kzIv<+1#B===z)#cY(K9Ts~9qy@Y3Hk9<~E&*oe*gsb0)5Z2xBhHr42K zrgG36`I5j4&Q0`~8nN-=mHm8mF@qjDV^-aYH02IKg&7mozAP|XFJVUPjT~mx#SCJ$ zzyxNkbexf|p!R@3P3{YbjmsaDmZlfO2~vZotuTRF2cfnrItG)OmAxpFv{51f8!tRQ zN^E>LtP;8EVg`}xHi6tMLT;}$n#wsZ3iE;9>(zEgY!aB%H|YH8>`~ zT_hlr^ABRBTC_S|;+3ZHWJX01#wW*D+dOhDF=8XL0`sc7Cx!wQ&6 z|748q<7)pTHakYpiy0mx958|0l5(}dJq?Qn_g)~GZlb6LlGyC1p}LsCK$kKBYGQPJ8)Z`rH9fIA#p~i?2yX?i~ zl2=VR=RIm&5}VCA^=cb&3|D2l9o730`noFf z<0h0@vEtfCYmcApxw_Xze(4m3uL z*!Z>^VQa2m);xN+U$g-B?0RY=#D+C|>TFFfhKEwPPVlX|JHZd@s7>6qR)2f@k2Tp7AHow%1 z8OH6;CU9$;h~{$FX0rYg%&P=!a$Szt>;#uy%~#&yNY0LO?CyPPbQ^WHhBMB3V) z8nZHm{BSCrPWfwJT&-GSK?T+>7*>u*4_Lnu}5 zN{Nm8xs-uJFJ^eYX1KACy+{oivDqmuy_jLhE;j*LZS{sv2-xIx55#8c4SKP17iuK? zuuQ;KdI@Z0$YrLo-XbuQNA<*Jm#ON-JeAb?kE^aRf!SQ=#ks@L@#v_Pts&(d7XZow z(;EdFPeDr5Em9MUbEy;F;3 z3u^pcCfMwlfe|CNO0#Wv_|Diw-rrE+khs6>62!*hfzsggVxCdpo9eFCcbHIXYb+W+ zm>C)ImHb|-M(xFd%}#tUV#F>_GL+r4hKY|O;^uOXN^G1oR%%i&R&EpYsPC)0CLc1P zNjeXuvO{H}?O`=!#9qchrWf;6j_MED=j#X=j$!KWo*$@V3*9ly?5s7GIb`h~-;x=d zNLzVnjVPnwm#F=e*m$(D(kJv{f?BBS=;kI9dY$d8w7JYt?!J7~XWj5sLc#K|mDo5!Rw`I8 z=IJcz58uK%D%cIh^F&D5=RKg+hl|{xMv&NSDykPN4-btP;3*~$Y=&bqd`@#^)R^T2 zn=O_wV#LP5n_SL0J`wZPvu4Ge<(`$;?0O)*Si9RGP~sYdyslAX(1a5GZ>pJ4Lq=>i z&D4u|%IEcmY>N@dd{5F`qlS#wY;UR;GYr}8IzncqH(guj7S_L%{xTE&eP`ua;@%*~ z%tHLDFh6)7lV@!qvDu!gx|lmKtUq+oIx5%g>)Y~TVTQkv!^LW^-6hy;{g)9VcDdG6 zE*W+~Wq%wFw4r4qUY#;^l89)9Tm>dJt5dz$;>wQeAZnnl9oHcfI$fBvvWKkfJ}V6f z!a?A*fHlm`{WW?e%s=BZ)VAHaJ}064vnP5J80wyO~%Hd~F*i)V@-8GFM)DaT zZvWMu&fqfnTqf;r4snri3;CpOV&jrCWe27gGj-SY2jxqMFSGdUB}i&a!~>I&HZ7y_&fOl+rh$nrNm zy-;mq#AdT1y_hM;o-qVjhuE2yP91gVFOy65s8J*K8qU!4Vigl{bv3kUbIruE7I+WU zsLx#vE7gDz8=qVm{PkiL75%yb)@cZ^raim&?)PTiOd#u42Qp%_DWqP^6kLl9!PPYZ zCyI?`t=ty>j@hfWGh+91cBU7@o|bAyb+xn83<1~SEbFdeJ@;I-xe=Q^$VM+#v67{( zsP!5`t+R45yw^*aUZ78HY{X`J0liqoK(DT_onr{L7E$T-1=w0OV8q_Qd8b~?R9L&n z5MVPAuvEO3H#5vu`zNv4L&Nl96?+EiYH0fn;ntQ*7V?SAwbHgqrUzZGhK$%7IaAY% zl}DTID7cj-ue|opREFs4tE3t(v03}qi{Z&n?##xO2!37FnW~>U>;0mhtLxV^r!q7O z*K-xyQ!=-;nelyOc^9QdldD!& z*&T#xu{&P$@HFSYNQJyX@LpR-@akOaJzT%0IUao-T+hB0i0jt9tXn&K443aal7n4e zwfgms)qefoRs6cT*fIF^&&AEU0uvI%{x(Bb?t2at$zBSl3)L)z-&ehVq-~2X=Zw{ zHnWh;sUu`{4spM&p9eT0?pe5=JsqCco(04@056Dt9_vhlHycYFHjanvPb0z3 z-iaR8O_MiG7PrFt*>gSklD_VV)P$8zjaxmrY`lkFIF#$L4#(l&qgKVg*3y@2^SSQr z@a|)4vbcOOJ`hdBWAWY%R(vRy=-)Uv*f)|G*w{DN-@jp9-?~Y7dWiJJCTFc$aiM!z zdP4)k8u9kgl(OSv=wJS1GCY1-pWo! zS_;MK7D&Nw?QQ@)vd&(MAJ7>TyKE1jHj(=w!1X5e@(g@qvA?mtz0Xb1NojO zau9n9*Gjz@9yQ=P^q!y{`ice^$HJA?IP@)XE76@hG9K@SV4pAK{HdT%tpZ|i=Tx8< z!@)q~R502A70h2bYUR62y|beMp9<(vD}dPNathFk;RdtDDInJX1)St4U?Q721n;fS zc8@`q2ZvVIqKIC#B8a_{Q-od&SK2gA5!W?95%X6%F$m^RU3c*539Ho#AU5j>da=fO z!mSNZz&xRVRIFC~*QoI)_Su{R=*4iSVdEU&4UNP9nz35(KTC~2v041}VvWWB?G3>H zL}CAczuX)6j6>aOJrMh1&JXlrxWcz_e(?U_^nm+y10P234et4{?BR@jW>%gL;cryd zAuLNSV5|~sc2vfQ5gY5l6=*4io zuxs<5L={oxJ&@AM7aL#`HRHZOjXSYf-1TCO#r-?~2)Li5#+}$K?s~Dt;{L=x0`8}( zaVIv5yI!oZxc{;dxO-lj?jyd>QR7Z*93&~@yI!oZxIg=kfcs)K?!;zs*NZh4_v!P^ zPg%(2Z|`e9ec1alHSWY_ao3AA7WY~I2)Li2#+}$K?s~Dt;(lTSaIbtlrVneMsYadH z_-1Eibk~cO#}zQ53p{&Y8Gh8~t5GL5?%`6Rt`}=8>g&Rf`iW}PiOr&}7i%o)&kH~5 zr>IdUHg1en+PYq>v8YEHfO_jh^hi1rg}3+mi0ZS{m=hb{-mAo1FIHYU;TE#mof-V? zsca5jh3LOsbEH{&y6=iA?PA0{v-X#wYJ#j4T^*b);0xFT?@ccY?)hA^s>#+Ds%@Rv zm;fqmT`yL)^+sTx2|MOX)tD2T#au7eSj=A%cFa#xV@_-qbG=w&F@JRfFt2!{s*et9 zwi(~bFsWM~nO=^oLb~gvNUaY)tB+#PHlYzqR>{G;r zZ}u1AzV>&JJOM=P7res%7Oe74pPr*vQm-w4Te*!e+TU*t^!E3TBnCGO^$lzs8jSU= z8|YuRZm@r2f1+RBMmW!@`0WvgaqhchCm#PWsn|oHrs#2 z4{i_PpWsUj6V7kfi{HZL^1mtAr$~DUOJ;O!J-6tUk*$h5?%aD8+6nP=qAol4YP$Eb z;-yBcX|hm?+`orsD=v*^t!UobLt83J(R*0zteCW)K>*t9Kfp(%)BYoTpf){PeW}nY z1aFU4?{^KueoiAM8v|VpCmW7mfEzZQ^^Q(kXIY0+xjbz6P`0j1-MU*vGvM_hp4Jij z3b>83b$T&eZ!KB}HODq+W*BUB9=4+~wraP+y~RdGM^jy6Q8+R<4$tGUwO8AZKDtteuzR6t>Q?G7_qVX96CoDY`A{haepI|l=eo$;8Y3bKr8H}NX|qu00dJ7r$3lG z0;g4!dV|eJwqz3U?wQI^!=x6!6E-WS7Jn3M92YA41QL@|3u5EkwVYZUb!L5K}(7?#xCjw&n)8g00OLeWueZ!`=}M>PzsurEhq0df1NCD=#p z&TetW!^4{K3)5Gc#NTIoHwhs97p>H!>Bv4%-oqo=Q6go~bTYd`u_`HfVZ^6~xBF5tNG3i-l5Aw}wbj zQy2q(SnW~7W^+Him?!rOi$}d7Jc_Cz_xplcKi?2+wwT0-5!;jdg+)Ja508Fo$^E{i zR?wq@-NZ#YMvT~=+%GH&dVj+e6pGyMP@`dRa=({BE9{$*+=1jCB<}^{W`QSS7I-YP zA^#;TSWW|fBiL*jNMdptNNk+Smeatglq`Os_Bvv--IiX=(`|hczzjno_(g0-I1<4c zx~*TT_46yij&V_u5hJ#z+X{<*z7rn()Y5G|tyU1R_j3x;i+Q@Ouqfz>hASu(-PWfY z4TIBdeI8n2e;dhsfK$x=JQT<%7*^Y|DW=KO+V&iH# z(NVdLY#S463E=P~A2q=rsvYb}!DbV8MvT~a0I}SC9=~&!w?VDyA^8*H&Tx9IXGl(L zjIGKcxn8WrZ4j&<_yd|)2x=$O7KWzFoW66nKkfWftsP=xU{z{IFV-4H?R17nJG(B} z=gW6~tkw>(am~C^J9@EDYG-knv~xMEd#((%Ol13!T06wXZQDxi=*2>*ozudjo#XfK z_NAPksFg!(w%gH*g|ge}ZMbr9bIPwm(g{C{?P%z@H5!IopZ2k zN3Ffiq6BFKq8=9wcOEBnBn^y)=MtYA8M%Ve(8%R42$h<4g-T7caTCM^>2zrz>q}LU z7Nw>l{esOYswB2h8q(3#&T!RKUU}|0JjI8bw~7|Xi_VGN!@0R$47Ze#o3F56fa(i@ zx5pc8MrBuLmBi$~Z9++BOJ+dqZJd&-i-k@}neZrShLuT(XzWdcnf~%*p`&(5N5p0) zEc9aY!stjZ36GANt!y@v_3DRNu(C$z2i~OXsg4mF>u$<1WOcF7`OvGwtsf@F^$7jI ztA0KDA@+HkeyWRwPCvJYM?Y<;SPdcW6m^IrHtvH{x{zKhl)1v);Zf04CeP=o6-De` zPEmR>JS;bKzI6XTwxYV#iXt|vD7{!H74<-P6jjSCdb?1NJUp5$*z6=9BS!4=d_`XZncE&L!!^Ix7A9-%(v57_@yYL}ugUn$mEeQP3>4f{4x5 z#PwqBVN}rL;Ze{Gc)wQM%H=G7PSl~+5wY0;fnLnBf+;Lc^z-oOs5N&mH8J5+LtSbO z5t|Kjda+Q3x!;CJLxl-=VX{vR%~WfM*sO;1VxiQ~-x{tVocjw)E%kFDKqCa%$qO3| zgERLRg;v;ONRA-65y@>x?m}`el8++6?#TWMlE;wz49TyN`~`?R4du*lY}>uHw)u^% zA`nP3QIRVJo1NbvF?lUHvGE<1V#Ha_lWVB>T_$cmQ>rKt8|x0raz%A9cNE;YC>&Yx zB5X%Pr{Z1nQR#SCeO&#B?jPn$)j9(o6w%%sK+wF?m&r-_v= zR9!4|D(VTniu7J|p4yAHb1EV)(ujplMQ4XcMKelXB$WdgH`*@TNUkCgdnczOy%;V^ z6*_W*Z%Z_uPaTpDtYPeiE`F|LH^kmirA>_37Pn21tZqYng&<4Y({Q`NcdUdZSGcg@ zXt?IuF_sx0&D3z7+cvcy5qlM9vwAVl9D7)_bX9n?R7<)1LbZa3{UT05da+|Jm(%6D zrBFv~b}6G?Y<*>@ONAuV)$9tG7{E+-u=Q|sug zDmr4sFl4)VW>}o47#=j75k!(bQLsfehk^@MvC$h6h zUXJ7zB(F#EHX!csWftxP*s-~`LiyDqLzWBWFBWXJP)=fUp`6$!yCGmoh4P%Zxm+kG zHd`pKF6MR|9|7FLPzv}Aw&NeJP@Wh1*&uc7#6FwrHmZw-PCs7_kA7+^lv`>SA~stn zuPzok6@4%CDq;$rlGyAfn(AVqQ_)l5QBjSB@|bWVxlm4Q_Eb^5Sh-LRDPdT8 zqThr^M>UsWqiQV?n=QlY#XM!$uxRP8;n7kpW!Sh{LBwXuuzIm$F2mC0*)ptN z%u|MKUL1}zb2hdk9BF1t4P{tDTCS%On=QlY#XM!$uqf!H@F=LpGAw~D*HekjmSOc` zo-%A$bhIiwI;yD*OHj-8RARGbSiP913>y{=tq+lgYAM4K)N&b?*lZbAFXk!3hDAf? zH(Wzlh7C(L*BSyeLQsbNSfi6#rwsckXodY{B#$C_63Np@{)_}2$nHY21j!jl`jMOm z#I3;27GA#(>ahFr(R@K#jio*|C-Nte9ZMzH;r1%0=9J{*8Z2F|yja<7UJb`7Kc~#t zhg+#l>_$ViT!h%3-AH=?xiHjVufTRRG&v5%xnX-74TIx_mqRO@H%0#rUVQ7kAJ)b< nQQ7!P0viu_c8;5~&&O>7JCWcH6lY$|iRv~QmGegF!rK22RF&Q+ delta 22 dcmez0!hLBTTf-DaK4zPKhOL5(_WewydH`WS2QUBt diff --git a/apps/data-ingestion/src/index.ts b/apps/data-ingestion/src/index.ts index dddc0d6..aa3d651 100644 --- a/apps/data-ingestion/src/index.ts +++ b/apps/data-ingestion/src/index.ts @@ -4,9 +4,9 @@ */ // Framework imports +import { initializeServiceConfig } from '@stock-bot/config'; import { Hono } from 'hono'; import { cors } from 'hono/cors'; -import { initializeServiceConfig } from '@stock-bot/config'; // Library imports import { createServiceContainer, @@ -17,8 +17,8 @@ import { getLogger, setLoggerConfig, shutdownLoggers } from '@stock-bot/logger'; import { Shutdown } from '@stock-bot/shutdown'; import { handlerRegistry } from '@stock-bot/types'; // Local imports -import { createRoutes } from './routes/create-routes'; import { initializeAllHandlers } from './handlers'; +import { createRoutes } from './routes/create-routes'; const config = initializeServiceConfig(); console.log('Data Service Configuration:', JSON.stringify(config, null, 2)); @@ -123,7 +123,11 @@ async function initializeServices() { let totalScheduledJobs = 0; for (const [handlerName, config] of allHandlers) { if (config.scheduledJobs && config.scheduledJobs.length > 0) { - const queueManager = container!.resolve('queueManager'); + const queueManager = container.resolve('queueManager'); + if(!queueManager) { + logger.error('Queue manager is not initialized, cannot create scheduled jobs'); + continue; + } const queue = queueManager.getQueue(handlerName); for (const scheduledJob of config.scheduledJobs) { diff --git a/apps/data-pipeline/src/clients.ts b/apps/data-pipeline/src/clients.ts index 8cd54e2..5488cf8 100644 --- a/apps/data-pipeline/src/clients.ts +++ b/apps/data-pipeline/src/clients.ts @@ -1,27 +1,8 @@ -import { MongoDBClient } from '@stock-bot/mongodb'; -import { PostgreSQLClient } from '@stock-bot/postgres'; +/** + * Client exports for backward compatibility + * + * @deprecated Use ServiceContainer parameter instead + * This file will be removed once all operations are migrated + */ -let postgresClient: PostgreSQLClient | null = null; -let mongodbClient: MongoDBClient | null = null; - -export function setPostgreSQLClient(client: PostgreSQLClient): void { - postgresClient = client; -} - -export function getPostgreSQLClient(): PostgreSQLClient { - if (!postgresClient) { - throw new Error('PostgreSQL client not initialized. Call setPostgreSQLClient first.'); - } - return postgresClient; -} - -export function setMongoDBClient(client: MongoDBClient): void { - mongodbClient = client; -} - -export function getMongoDBClient(): MongoDBClient { - if (!mongodbClient) { - throw new Error('MongoDB client not initialized. Call setMongoDBClient first.'); - } - return mongodbClient; -} +export { getMongoDBClient, getPostgreSQLClient } from './migration-helper'; \ No newline at end of file diff --git a/apps/data-pipeline/src/container-setup.ts b/apps/data-pipeline/src/container-setup.ts new file mode 100644 index 0000000..a1f2638 --- /dev/null +++ b/apps/data-pipeline/src/container-setup.ts @@ -0,0 +1,34 @@ +/** + * Service Container Setup for Data Pipeline + * Configures dependency injection for the data pipeline service + */ + +import type { ServiceContainer } from '@stock-bot/di'; +import { getLogger } from '@stock-bot/logger'; +import type { AppConfig } from '@stock-bot/config'; + +const logger = getLogger('data-pipeline-container'); + +/** + * Configure the service container for data pipeline workloads + */ +export function setupServiceContainer( + config: AppConfig, + container: ServiceContainer +): ServiceContainer { + logger.info('Configuring data pipeline service container...'); + + // Data pipeline specific configuration + // This service does more complex queries and transformations + const poolSizes = { + mongodb: config.environment === 'production' ? 40 : 20, + postgres: config.environment === 'production' ? 50 : 25, + cache: config.environment === 'production' ? 30 : 15, + }; + + logger.info('Data pipeline pool sizes configured', poolSizes); + + // The container is already configured with connections + // Just return it with our logging + return container; +} \ No newline at end of file diff --git a/apps/data-pipeline/src/handlers/exchanges/exchanges.handler.ts b/apps/data-pipeline/src/handlers/exchanges/exchanges.handler.ts index 2968dd2..cd503c3 100644 --- a/apps/data-pipeline/src/handlers/exchanges/exchanges.handler.ts +++ b/apps/data-pipeline/src/handlers/exchanges/exchanges.handler.ts @@ -1,5 +1,6 @@ import { getLogger } from '@stock-bot/logger'; -import { handlerRegistry, type HandlerConfig, type ScheduledJobConfig } from '@stock-bot/queue'; +import { handlerRegistry, createJobHandler, type HandlerConfig, type ScheduledJobConfig } from '@stock-bot/queue'; +import type { ServiceContainer } from '@stock-bot/di'; import { exchangeOperations } from './operations'; const logger = getLogger('exchanges-handler'); @@ -51,8 +52,23 @@ const exchangesHandlerConfig: HandlerConfig = { }, }; -export function initializeExchangesHandler(): void { +export function initializeExchangesHandler(container: ServiceContainer) { logger.info('Registering exchanges handler...'); - handlerRegistry.registerHandler(HANDLER_NAME, exchangesHandlerConfig); + + // Update operations to use container + const containerAwareOperations = Object.entries(exchangeOperations).reduce((acc, [key, operation]) => { + acc[key] = createJobHandler(async (payload: any) => { + return operation(payload, container); + }); + return acc; + }, {} as Record); + + const exchangesHandlerConfigWithContainer: HandlerConfig = { + ...exchangesHandlerConfig, + operations: containerAwareOperations, + }; + + handlerRegistry.register(HANDLER_NAME, exchangesHandlerConfigWithContainer); logger.info('Exchanges handler registered successfully'); } + diff --git a/apps/data-pipeline/src/handlers/exchanges/operations/clear-postgresql-data.operations.ts b/apps/data-pipeline/src/handlers/exchanges/operations/clear-postgresql-data.operations.ts index 808320e..b47b0cb 100644 --- a/apps/data-pipeline/src/handlers/exchanges/operations/clear-postgresql-data.operations.ts +++ b/apps/data-pipeline/src/handlers/exchanges/operations/clear-postgresql-data.operations.ts @@ -1,10 +1,13 @@ import { getLogger } from '@stock-bot/logger'; -import { getPostgreSQLClient } from '../../../clients'; +import type { ServiceContainer } from '@stock-bot/di'; import type { JobPayload } from '../../../types/job-payloads'; const logger = getLogger('enhanced-sync-clear-postgresql-data'); -export async function clearPostgreSQLData(payload: JobPayload): Promise<{ +export async function clearPostgreSQLData( + payload: JobPayload, + container: ServiceContainer +): Promise<{ exchangesCleared: number; symbolsCleared: number; mappingsCleared: number; @@ -12,7 +15,7 @@ export async function clearPostgreSQLData(payload: JobPayload): Promise<{ logger.info('Clearing existing PostgreSQL data...'); try { - const postgresClient = getPostgreSQLClient(); + const postgresClient = container.postgres; // Start transaction for atomic operations await postgresClient.query('BEGIN'); @@ -50,7 +53,7 @@ export async function clearPostgreSQLData(payload: JobPayload): Promise<{ return { exchangesCleared, symbolsCleared, mappingsCleared }; } catch (error) { - const postgresClient = getPostgreSQLClient(); + const postgresClient = container.postgres; await postgresClient.query('ROLLBACK'); logger.error('Failed to clear PostgreSQL data', { error }); throw error; diff --git a/apps/data-pipeline/src/handlers/exchanges/operations/enhanced-sync-status.operations.ts b/apps/data-pipeline/src/handlers/exchanges/operations/enhanced-sync-status.operations.ts index da188e9..f1ab881 100644 --- a/apps/data-pipeline/src/handlers/exchanges/operations/enhanced-sync-status.operations.ts +++ b/apps/data-pipeline/src/handlers/exchanges/operations/enhanced-sync-status.operations.ts @@ -1,14 +1,17 @@ import { getLogger } from '@stock-bot/logger'; -import { getPostgreSQLClient } from '../../../clients'; +import type { ServiceContainer } from '@stock-bot/di'; import type { JobPayload, SyncStatus } from '../../../types/job-payloads'; const logger = getLogger('enhanced-sync-status'); -export async function getSyncStatus(payload: JobPayload): Promise { +export async function getSyncStatus( + payload: JobPayload, + container: ServiceContainer +): Promise { logger.info('Getting comprehensive sync status...'); try { - const postgresClient = getPostgreSQLClient(); + const postgresClient = container.postgres; const query = ` SELECT provider, data_type as "dataType", last_sync_at as "lastSyncAt", last_sync_count as "lastSyncCount", sync_errors as "syncErrors" diff --git a/apps/data-pipeline/src/handlers/exchanges/operations/exchange-stats.operations.ts b/apps/data-pipeline/src/handlers/exchanges/operations/exchange-stats.operations.ts index 2c79d96..fdc17fc 100644 --- a/apps/data-pipeline/src/handlers/exchanges/operations/exchange-stats.operations.ts +++ b/apps/data-pipeline/src/handlers/exchanges/operations/exchange-stats.operations.ts @@ -1,14 +1,17 @@ import { getLogger } from '@stock-bot/logger'; -import { getPostgreSQLClient } from '../../../clients'; +import type { ServiceContainer } from '@stock-bot/di'; import type { JobPayload } from '../../../types/job-payloads'; const logger = getLogger('enhanced-sync-exchange-stats'); -export async function getExchangeStats(payload: JobPayload): Promise { +export async function getExchangeStats( + payload: JobPayload, + container: ServiceContainer +): Promise { logger.info('Getting exchange statistics...'); try { - const postgresClient = getPostgreSQLClient(); + const postgresClient = container.postgres; const query = ` SELECT COUNT(*) as total_exchanges, diff --git a/apps/data-pipeline/src/handlers/exchanges/operations/provider-mapping-stats.operations.ts b/apps/data-pipeline/src/handlers/exchanges/operations/provider-mapping-stats.operations.ts index 62cb229..9d07412 100644 --- a/apps/data-pipeline/src/handlers/exchanges/operations/provider-mapping-stats.operations.ts +++ b/apps/data-pipeline/src/handlers/exchanges/operations/provider-mapping-stats.operations.ts @@ -1,14 +1,17 @@ import { getLogger } from '@stock-bot/logger'; -import { getPostgreSQLClient } from '../../../clients'; +import type { ServiceContainer } from '@stock-bot/di'; import type { JobPayload } from '../../../types/job-payloads'; const logger = getLogger('enhanced-sync-provider-mapping-stats'); -export async function getProviderMappingStats(payload: JobPayload): Promise { +export async function getProviderMappingStats( + payload: JobPayload, + container: ServiceContainer +): Promise { logger.info('Getting provider mapping statistics...'); try { - const postgresClient = getPostgreSQLClient(); + const postgresClient = container.postgres; const query = ` SELECT provider, diff --git a/apps/data-pipeline/src/handlers/exchanges/operations/sync-all-exchanges.operations.ts b/apps/data-pipeline/src/handlers/exchanges/operations/sync-all-exchanges.operations.ts index e6ba7fd..7c642b4 100644 --- a/apps/data-pipeline/src/handlers/exchanges/operations/sync-all-exchanges.operations.ts +++ b/apps/data-pipeline/src/handlers/exchanges/operations/sync-all-exchanges.operations.ts @@ -1,10 +1,10 @@ import { getLogger } from '@stock-bot/logger'; -import { getMongoDBClient, getPostgreSQLClient } from '../../../clients'; +import type { ServiceContainer } from '@stock-bot/di'; import type { JobPayload, SyncResult } from '../../../types/job-payloads'; const logger = getLogger('enhanced-sync-all-exchanges'); -export async function syncAllExchanges(payload: JobPayload): Promise { +export async function syncAllExchanges(payload: JobPayload, container: ServiceContainer): Promise { const clearFirst = payload.clearFirst || true; logger.info('Starting comprehensive exchange sync...', { clearFirst }); @@ -17,7 +17,7 @@ export async function syncAllExchanges(payload: JobPayload): Promise }; try { - const postgresClient = getPostgreSQLClient(); + const postgresClient = container.postgres; // Clear existing data if requested if (clearFirst) { @@ -28,11 +28,11 @@ export async function syncAllExchanges(payload: JobPayload): Promise await postgresClient.query('BEGIN'); // 1. Sync from EOD exchanges (comprehensive global data) - const eodResult = await syncEODExchanges(); + const eodResult = await syncEODExchanges(container); mergeResults(result, eodResult); // 2. Sync from IB exchanges (detailed asset information) - const ibResult = await syncIBExchanges(); + const ibResult = await syncIBExchanges(container); mergeResults(result, ibResult); // 3. Update sync status @@ -43,13 +43,14 @@ export async function syncAllExchanges(payload: JobPayload): Promise logger.info('Comprehensive exchange sync completed', result); return result; } catch (error) { - const postgresClient = getPostgreSQLClient(); + const postgresClient = container.postgres; await postgresClient.query('ROLLBACK'); logger.error('Comprehensive exchange sync failed', { error }); throw error; } } + async function clearPostgreSQLData(postgresClient: any): Promise { logger.info('Clearing existing PostgreSQL data...'); @@ -66,8 +67,8 @@ async function clearPostgreSQLData(postgresClient: any): Promise { logger.info('PostgreSQL data cleared successfully'); } -async function syncEODExchanges(): Promise { - const mongoClient = getMongoDBClient(); +async function syncEODExchanges(container: ServiceContainer): Promise { + const mongoClient = container.mongodb; const exchanges = await mongoClient.find('eodExchanges', { active: true }); const result: SyncResult = { processed: 0, created: 0, updated: 0, skipped: 0, errors: 0 }; @@ -80,7 +81,8 @@ async function syncEODExchanges(): Promise { exchange.Name, exchange.CountryISO2, exchange.Currency, - 0.95 // very high confidence for EOD data + 0.95, // very high confidence for EOD data + container ); result.processed++; @@ -94,8 +96,8 @@ async function syncEODExchanges(): Promise { return result; } -async function syncIBExchanges(): Promise { - const mongoClient = getMongoDBClient(); +async function syncIBExchanges(container: ServiceContainer): Promise { + const mongoClient = container.mongodb; const exchanges = await mongoClient.find('ibExchanges', {}); const result: SyncResult = { processed: 0, created: 0, updated: 0, skipped: 0, errors: 0 }; @@ -108,7 +110,8 @@ async function syncIBExchanges(): Promise { exchange.name, exchange.country_code, 'USD', // IB doesn't specify currency, default to USD - 0.85 // good confidence for IB data + 0.85, // good confidence for IB data + container ); result.processed++; @@ -128,16 +131,17 @@ async function createProviderExchangeMapping( providerExchangeName: string, countryCode: string | null, currency: string | null, - confidence: number + confidence: number, + container: ServiceContainer ): Promise { if (!providerExchangeCode) { return; } - const postgresClient = getPostgreSQLClient(); + const postgresClient = container.postgres; // Check if mapping already exists - const existingMapping = await findProviderExchangeMapping(provider, providerExchangeCode); + const existingMapping = await findProviderExchangeMapping(provider, providerExchangeCode, container); if (existingMapping) { // Don't override existing mappings to preserve manual work return; @@ -148,7 +152,8 @@ async function createProviderExchangeMapping( providerExchangeCode, providerExchangeName, countryCode, - currency + currency, + container ); // Create the provider exchange mapping @@ -175,12 +180,13 @@ async function findOrCreateMasterExchange( providerCode: string, providerName: string, countryCode: string | null, - currency: string | null + currency: string | null, + container: ServiceContainer ): Promise { - const postgresClient = getPostgreSQLClient(); + const postgresClient = container.postgres; // First, try to find exact match - let masterExchange = await findExchangeByCode(providerCode); + let masterExchange = await findExchangeByCode(providerCode, container); if (masterExchange) { return masterExchange; @@ -189,7 +195,7 @@ async function findOrCreateMasterExchange( // Try to find by similar codes (basic mapping) const basicMapping = getBasicExchangeMapping(providerCode); if (basicMapping) { - masterExchange = await findExchangeByCode(basicMapping); + masterExchange = await findExchangeByCode(basicMapping, container); if (masterExchange) { return masterExchange; } @@ -230,17 +236,18 @@ function getBasicExchangeMapping(providerCode: string): string | null { async function findProviderExchangeMapping( provider: string, - providerExchangeCode: string + providerExchangeCode: string, + container: ServiceContainer ): Promise { - const postgresClient = getPostgreSQLClient(); + const postgresClient = container.postgres; const query = 'SELECT * FROM provider_exchange_mappings WHERE provider = $1 AND provider_exchange_code = $2'; const result = await postgresClient.query(query, [provider, providerExchangeCode]); return result.rows[0] || null; } -async function findExchangeByCode(code: string): Promise { - const postgresClient = getPostgreSQLClient(); +async function findExchangeByCode(code: string, container: ServiceContainer): Promise { + const postgresClient = container.postgres; const query = 'SELECT * FROM exchanges WHERE code = $1'; const result = await postgresClient.query(query, [code]); return result.rows[0] || null; diff --git a/apps/data-pipeline/src/handlers/index.ts b/apps/data-pipeline/src/handlers/index.ts new file mode 100644 index 0000000..a596fb4 --- /dev/null +++ b/apps/data-pipeline/src/handlers/index.ts @@ -0,0 +1,33 @@ +/** + * Handler initialization for data pipeline service + * Registers all handlers with the service container + */ + +import type { ServiceContainer } from '@stock-bot/di'; +import { getLogger } from '@stock-bot/logger'; +import { initializeExchangesHandler } from './exchanges/exchanges.handler'; +import { initializeSymbolsHandler } from './symbols/symbols.handler'; + +const logger = getLogger('pipeline-handler-init'); + +/** + * Initialize all handlers with the service container + */ +export async function initializeAllHandlers(container: ServiceContainer): Promise { + logger.info('Initializing data pipeline handlers...'); + + try { + // Initialize exchanges handler with container + initializeExchangesHandler(container); + logger.debug('Exchanges handler initialized'); + + // Initialize symbols handler with container + initializeSymbolsHandler(container); + logger.debug('Symbols handler initialized'); + + logger.info('All pipeline handlers initialized successfully'); + } catch (error) { + logger.error('Failed to initialize handlers', { error }); + throw error; + } +} \ No newline at end of file diff --git a/apps/data-pipeline/src/handlers/symbols/symbols.handler.ts b/apps/data-pipeline/src/handlers/symbols/symbols.handler.ts index 9013e06..3cd384b 100644 --- a/apps/data-pipeline/src/handlers/symbols/symbols.handler.ts +++ b/apps/data-pipeline/src/handlers/symbols/symbols.handler.ts @@ -1,5 +1,6 @@ import { getLogger } from '@stock-bot/logger'; -import { handlerRegistry, type HandlerConfig, type ScheduledJobConfig } from '@stock-bot/queue'; +import { handlerRegistry, createJobHandler, type HandlerConfig, type ScheduledJobConfig } from '@stock-bot/queue'; +import type { ServiceContainer } from '@stock-bot/di'; import { symbolOperations } from './operations'; const logger = getLogger('symbols-handler'); @@ -34,8 +35,22 @@ const symbolsHandlerConfig: HandlerConfig = { }, }; -export function initializeSymbolsHandler(): void { +export function initializeSymbolsHandler(container: ServiceContainer): void { logger.info('Registering symbols handler...'); - handlerRegistry.registerHandler(HANDLER_NAME, symbolsHandlerConfig); + + // Update operations to use container + const containerAwareOperations = Object.entries(symbolOperations).reduce((acc, [key, operation]) => { + acc[key] = createJobHandler(async (payload: any) => { + return operation(payload, container); + }); + return acc; + }, {} as Record); + + const symbolsHandlerConfigWithContainer: HandlerConfig = { + ...symbolsHandlerConfig, + operations: containerAwareOperations, + }; + + handlerRegistry.register(HANDLER_NAME, symbolsHandlerConfigWithContainer); logger.info('Symbols handler registered successfully'); } diff --git a/apps/data-pipeline/src/index.ts b/apps/data-pipeline/src/index.ts index d5f487d..3ffc6dd 100644 --- a/apps/data-pipeline/src/index.ts +++ b/apps/data-pipeline/src/index.ts @@ -1,22 +1,31 @@ +/** + * Data Pipeline Service with Dependency Injection + * Uses Awilix container for managing database connections and services + */ + // Framework imports import { Hono } from 'hono'; import { cors } from 'hono/cors'; import { initializeServiceConfig } from '@stock-bot/config'; + // Library imports +import { + createServiceContainer, + initializeServices as initializeAwilixServices, + type ServiceContainer +} from '@stock-bot/di'; import { getLogger, setLoggerConfig, shutdownLoggers } from '@stock-bot/logger'; -import { MongoDBClient } from '@stock-bot/mongodb'; -import { PostgreSQLClient } from '@stock-bot/postgres'; -import { QueueManager, type QueueManagerConfig } from '@stock-bot/queue'; import { Shutdown } from '@stock-bot/shutdown'; -import { setMongoDBClient, setPostgreSQLClient } from './clients'; +import { handlerRegistry } from '@stock-bot/types'; + // Local imports -import { enhancedSyncRoutes, healthRoutes, statsRoutes, syncRoutes } from './routes'; +import { createRoutes } from './routes/create-routes'; +import { setupServiceContainer } from './container-setup'; +import { initializeAllHandlers } from './handlers'; const config = initializeServiceConfig(); -console.log('Data Sync Service Configuration:', JSON.stringify(config, null, 2)); +console.log('Data Pipeline Service Configuration:', JSON.stringify(config, null, 2)); const serviceConfig = config.service; -const databaseConfig = config.database; -const queueConfig = config.queue; if (config.log) { setLoggerConfig({ @@ -31,129 +40,91 @@ if (config.log) { // Create logger AFTER config is set const logger = getLogger('data-pipeline'); -const app = new Hono(); - -// Add CORS middleware -app.use( - '*', - cors({ - origin: '*', - allowMethods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'], - allowHeaders: ['Content-Type', 'Authorization'], - credentials: false, - }) -); const PORT = serviceConfig.port; let server: ReturnType | null = null; -let mongoClient: MongoDBClient | null = null; -let postgresClient: PostgreSQLClient | null = null; -let queueManager: QueueManager | null = null; +let container: ServiceContainer | null = null; +let app: Hono | null = null; // Initialize shutdown manager const shutdown = Shutdown.getInstance({ timeout: 15000 }); -// Mount routes -app.route('/health', healthRoutes); -app.route('/sync', syncRoutes); -app.route('/sync', enhancedSyncRoutes); -app.route('/sync/stats', statsRoutes); - -// Initialize services +// Initialize services with DI pattern async function initializeServices() { - logger.info('Initializing data sync service...'); + logger.info('Initializing data pipeline service with DI...'); try { - // Initialize MongoDB client - logger.debug('Connecting to MongoDB...'); - const mongoConfig = databaseConfig.mongodb; - mongoClient = new MongoDBClient( - { - uri: mongoConfig.uri, - database: mongoConfig.database, - host: mongoConfig.host || 'localhost', - port: mongoConfig.port || 27017, - timeouts: { - connectTimeout: 30000, - socketTimeout: 30000, - serverSelectionTimeout: 5000, - }, + // Create Awilix container with proper config structure + logger.debug('Creating Awilix DI container...'); + const awilixConfig = { + redis: { + host: config.database.dragonfly.host, + port: config.database.dragonfly.port, + db: config.database.dragonfly.db, }, - logger - ); - await mongoClient.connect(); - setMongoDBClient(mongoClient); - logger.info('MongoDB connected'); - - // Initialize PostgreSQL client - logger.debug('Connecting to PostgreSQL...'); - const pgConfig = databaseConfig.postgres; - postgresClient = new PostgreSQLClient( - { - host: pgConfig.host, - port: pgConfig.port, - database: pgConfig.database, - username: pgConfig.user, - password: pgConfig.password, - poolSettings: { - min: 2, - max: pgConfig.poolSize || 10, - idleTimeoutMillis: pgConfig.idleTimeout || 30000, - }, + mongodb: { + uri: config.database.mongodb.uri, + database: config.database.mongodb.database, }, - logger - ); - await postgresClient.connect(); - setPostgreSQLClient(postgresClient); - logger.info('PostgreSQL connected'); - - // Initialize queue system (with delayed worker start) - logger.debug('Initializing queue system...'); - const queueManagerConfig: QueueManagerConfig = { - redis: queueConfig?.redis || { - host: 'localhost', - port: 6379, - db: 1, + postgres: { + host: config.database.postgres.host, + port: config.database.postgres.port, + database: config.database.postgres.database, + user: config.database.postgres.user, + password: config.database.postgres.password, }, - defaultQueueOptions: { - defaultJobOptions: queueConfig?.defaultJobOptions || { - attempts: 3, - backoff: { - type: 'exponential', - delay: 1000, - }, - removeOnComplete: 10, - removeOnFail: 5, - }, - workers: 2, - concurrency: 1, - enableMetrics: true, - enableDLQ: true, + questdb: { + enabled: config.database.questdb.enabled || false, + host: config.database.questdb.host, + httpPort: config.database.questdb.httpPort, + pgPort: config.database.questdb.pgPort, + influxPort: config.database.questdb.ilpPort, + database: config.database.questdb.database, }, - enableScheduledJobs: true, - delayWorkerStart: true, // Prevent workers from starting until all singletons are ready }; + + container = createServiceContainer(awilixConfig); + await initializeAwilixServices(container); + logger.info('Awilix container created and initialized'); + + // Setup service-specific configuration + const serviceContainer = setupServiceContainer(config, container.resolve('serviceContainer')); + + // Initialize migration helper for backward compatibility + const { setContainerForMigration } = await import('./migration-helper'); + setContainerForMigration(serviceContainer); + logger.info('Migration helper initialized for backward compatibility'); + + // Create app with routes + app = new Hono(); + + // Add CORS middleware + app.use( + '*', + cors({ + origin: '*', + allowMethods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'], + allowHeaders: ['Content-Type', 'Authorization'], + credentials: false, + }) + ); + + // Create and mount routes using the service container + const routes = createRoutes(serviceContainer); + app.route('/', routes); - queueManager = QueueManager.getOrInitialize(queueManagerConfig); - logger.info('Queue system initialized'); - - // Initialize handlers (register handlers and scheduled jobs) - logger.debug('Initializing sync handlers...'); - const { initializeExchangesHandler } = await import('./handlers/exchanges/exchanges.handler'); - const { initializeSymbolsHandler } = await import('./handlers/symbols/symbols.handler'); - - initializeExchangesHandler(); - initializeSymbolsHandler(); - - logger.info('Sync handlers initialized'); + // Initialize handlers with service container + logger.debug('Initializing pipeline handlers with DI pattern...'); + await initializeAllHandlers(serviceContainer); + logger.info('Pipeline handlers initialized with DI pattern'); // Create scheduled jobs from registered handlers logger.debug('Creating scheduled jobs from registered handlers...'); - const { handlerRegistry } = await import('@stock-bot/queue'); - const allHandlers = handlerRegistry.getAllHandlers(); + const allHandlers = handlerRegistry.getAllHandlersWithSchedule(); let totalScheduledJobs = 0; for (const [handlerName, config] of allHandlers) { if (config.scheduledJobs && config.scheduledJobs.length > 0) { + const queueManager = container!.resolve('queueManager'); const queue = queueManager.getQueue(handlerName); for (const scheduledJob of config.scheduledJobs) { @@ -161,7 +132,7 @@ async function initializeServices() { const jobData = { handler: handlerName, operation: scheduledJob.operation, - payload: scheduledJob.payload || {}, + payload: scheduledJob.payload, }; // Build job options from scheduled job config @@ -192,14 +163,22 @@ async function initializeServices() { } logger.info('Scheduled jobs created', { totalJobs: totalScheduledJobs }); - // Now that all singletons are initialized and jobs are scheduled, start the workers + // Start queue workers logger.debug('Starting queue workers...'); - queueManager.startAllWorkers(); - logger.info('Queue workers started'); + const queueManager = container.resolve('queueManager'); + if (queueManager) { + queueManager.startAllWorkers(); + logger.info('Queue workers started'); + } logger.info('All services initialized successfully'); } catch (error) { - logger.error('Failed to initialize services', { error }); + console.error('DETAILED ERROR:', error); + logger.error('Failed to initialize services', { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + details: JSON.stringify(error, null, 2) + }); throw error; } } @@ -208,13 +187,17 @@ async function initializeServices() { async function startServer() { await initializeServices(); + if (!app) { + throw new Error('App not initialized'); + } + server = Bun.serve({ port: PORT, fetch: app.fetch, development: config.environment === 'development', }); - logger.info(`Data Sync Service started on port ${PORT}`); + logger.info(`Data pipeline service started on port ${PORT}`); } // Register shutdown handlers with priorities @@ -222,6 +205,7 @@ async function startServer() { shutdown.onShutdownHigh(async () => { logger.info('Shutting down queue system...'); try { + const queueManager = container?.resolve('queueManager'); if (queueManager) { await queueManager.shutdown(); } @@ -244,21 +228,27 @@ shutdown.onShutdownHigh(async () => { } }, 'HTTP Server'); -// Priority 2: Database connections (medium priority) +// Priority 2: Services and connections (medium priority) shutdown.onShutdownMedium(async () => { - logger.info('Disconnecting from databases...'); + logger.info('Disposing services and connections...'); try { - if (mongoClient) { - await mongoClient.disconnect(); + if (container) { + // Disconnect database clients + const mongoClient = container.resolve('mongoClient'); + if (mongoClient?.disconnect) await mongoClient.disconnect(); + + const postgresClient = container.resolve('postgresClient'); + if (postgresClient?.disconnect) await postgresClient.disconnect(); + + const questdbClient = container.resolve('questdbClient'); + if (questdbClient?.disconnect) await questdbClient.disconnect(); + + logger.info('All services disposed successfully'); } - if (postgresClient) { - await postgresClient.disconnect(); - } - logger.info('Database connections closed'); } catch (error) { - logger.error('Error closing database connections', { error }); + logger.error('Error disposing services', { error }); } -}, 'Databases'); +}, 'Services'); // Priority 3: Logger shutdown (lowest priority - runs last) shutdown.onShutdownLow(async () => { @@ -273,8 +263,8 @@ shutdown.onShutdownLow(async () => { // Start the service startServer().catch(error => { - logger.fatal('Failed to start data sync service', { error }); + logger.fatal('Failed to start data pipeline service', { error }); process.exit(1); }); -logger.info('Data sync service startup initiated'); +logger.info('Data pipeline service startup initiated with DI pattern'); \ No newline at end of file diff --git a/apps/data-pipeline/src/migration-helper.ts b/apps/data-pipeline/src/migration-helper.ts new file mode 100644 index 0000000..d0af885 --- /dev/null +++ b/apps/data-pipeline/src/migration-helper.ts @@ -0,0 +1,37 @@ +/** + * Temporary migration helper for data-pipeline service + * Provides backward compatibility while migrating to DI container + * + * TODO: Remove this file once all operations are migrated to use ServiceContainer + */ + +import type { ServiceContainer } from '@stock-bot/di'; +import type { MongoDBClient } from '@stock-bot/mongodb'; +import type { PostgreSQLClient } from '@stock-bot/postgres'; + +let containerInstance: ServiceContainer | null = null; + +export function setContainerForMigration(container: ServiceContainer): void { + containerInstance = container; +} + +export function getMongoDBClient(): MongoDBClient { + if (!containerInstance) { + throw new Error('Container not initialized. This is a migration helper - please update the operation to accept ServiceContainer parameter'); + } + return containerInstance.mongodb; +} + +export function getPostgreSQLClient(): PostgreSQLClient { + if (!containerInstance) { + throw new Error('Container not initialized. This is a migration helper - please update the operation to accept ServiceContainer parameter'); + } + return containerInstance.postgres; +} + +export function getQuestDBClient(): any { + if (!containerInstance) { + throw new Error('Container not initialized. This is a migration helper - please update the operation to accept ServiceContainer parameter'); + } + return containerInstance.questdb; +} \ No newline at end of file diff --git a/apps/data-pipeline/src/routes/create-routes.ts b/apps/data-pipeline/src/routes/create-routes.ts new file mode 100644 index 0000000..8cf160f --- /dev/null +++ b/apps/data-pipeline/src/routes/create-routes.ts @@ -0,0 +1,26 @@ +/** + * Route factory for data pipeline service + * Creates routes with access to the service container + */ + +import { Hono } from 'hono'; +import type { ServiceContainer } from '@stock-bot/di'; +import { healthRoutes, syncRoutes, enhancedSyncRoutes, statsRoutes } from './index'; + +export function createRoutes(container: ServiceContainer): Hono { + const app = new Hono(); + + // Add container to context for all routes + app.use('*', async (c, next) => { + c.set('container', container); + await next(); + }); + + // Mount routes + app.route('/health', healthRoutes); + app.route('/sync', syncRoutes); + app.route('/sync', enhancedSyncRoutes); + app.route('/sync/stats', statsRoutes); + + return app; +} \ No newline at end of file diff --git a/apps/web-api/src/clients.ts b/apps/web-api/src/clients.ts index 8cd54e2..bf8e03d 100644 --- a/apps/web-api/src/clients.ts +++ b/apps/web-api/src/clients.ts @@ -1,27 +1,8 @@ -import { MongoDBClient } from '@stock-bot/mongodb'; -import { PostgreSQLClient } from '@stock-bot/postgres'; +/** + * Client exports for backward compatibility + * + * @deprecated Use ServiceContainer parameter instead + * This file will be removed once all routes and services are migrated + */ -let postgresClient: PostgreSQLClient | null = null; -let mongodbClient: MongoDBClient | null = null; - -export function setPostgreSQLClient(client: PostgreSQLClient): void { - postgresClient = client; -} - -export function getPostgreSQLClient(): PostgreSQLClient { - if (!postgresClient) { - throw new Error('PostgreSQL client not initialized. Call setPostgreSQLClient first.'); - } - return postgresClient; -} - -export function setMongoDBClient(client: MongoDBClient): void { - mongodbClient = client; -} - -export function getMongoDBClient(): MongoDBClient { - if (!mongodbClient) { - throw new Error('MongoDB client not initialized. Call setMongoDBClient first.'); - } - return mongodbClient; -} +export { getMongoDBClient, getPostgreSQLClient } from './migration-helper'; \ No newline at end of file diff --git a/apps/web-api/src/container-setup.ts b/apps/web-api/src/container-setup.ts new file mode 100644 index 0000000..2e71f0f --- /dev/null +++ b/apps/web-api/src/container-setup.ts @@ -0,0 +1,34 @@ +/** + * Service Container Setup for Web API + * Configures dependency injection for the web API service + */ + +import type { ServiceContainer } from '@stock-bot/di'; +import { getLogger } from '@stock-bot/logger'; +import type { AppConfig } from '@stock-bot/config'; + +const logger = getLogger('web-api-container'); + +/** + * Configure the service container for web API workloads + */ +export function setupServiceContainer( + config: AppConfig, + container: ServiceContainer +): ServiceContainer { + logger.info('Configuring web API service container...'); + + // Web API specific configuration + // This service mainly reads data, so smaller pool sizes are fine + const poolSizes = { + mongodb: config.environment === 'production' ? 20 : 10, + postgres: config.environment === 'production' ? 30 : 15, + cache: config.environment === 'production' ? 20 : 10, + }; + + logger.info('Web API pool sizes configured', poolSizes); + + // The container is already configured with connections + // Just return it with our logging + return container; +} \ No newline at end of file diff --git a/apps/web-api/src/index.ts b/apps/web-api/src/index.ts index 4d247a4..76f5c16 100644 --- a/apps/web-api/src/index.ts +++ b/apps/web-api/src/index.ts @@ -1,125 +1,137 @@ /** - * Stock Bot Web API - REST API service for web application + * Stock Bot Web API with Dependency Injection + * REST API service using Awilix container for managing connections */ + +// Framework imports import { Hono } from 'hono'; import { cors } from 'hono/cors'; import { initializeServiceConfig } from '@stock-bot/config'; + +// Library imports +import { + createServiceContainer, + initializeServices as initializeAwilixServices, + type ServiceContainer +} from '@stock-bot/di'; import { getLogger, setLoggerConfig, shutdownLoggers } from '@stock-bot/logger'; -import { MongoDBClient } from '@stock-bot/mongodb'; -import { PostgreSQLClient } from '@stock-bot/postgres'; import { Shutdown } from '@stock-bot/shutdown'; -import { exchangeRoutes } from './routes/exchange.routes'; -import { healthRoutes } from './routes/health.routes'; -// Import routes -import { setMongoDBClient, setPostgreSQLClient } from './clients'; -// Initialize configuration with automatic monorepo config inheritance -const config = await initializeServiceConfig(); +// Local imports +import { createRoutes } from './routes/create-routes'; +import { setupServiceContainer } from './container-setup'; + +const config = initializeServiceConfig(); +console.log('Web API Service Configuration:', JSON.stringify(config, null, 2)); const serviceConfig = config.service; -const databaseConfig = config.database; -// Initialize logger with config -const loggingConfig = config.logging; -if (loggingConfig) { +if (config.log) { setLoggerConfig({ - logLevel: loggingConfig.level, + logLevel: config.log.level, logConsole: true, logFile: false, environment: config.environment, + hideObject: config.log.hideObject, }); } -const app = new Hono(); - -// Add CORS middleware -app.use( - '*', - cors({ - origin: ['http://localhost:4200', 'http://localhost:3000', 'http://localhost:3002'], // React dev server ports - allowMethods: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'], - allowHeaders: ['Content-Type', 'Authorization'], - credentials: true, - }) -); - +// Create logger AFTER config is set const logger = getLogger('web-api'); + const PORT = serviceConfig.port; let server: ReturnType | null = null; -let postgresClient: PostgreSQLClient | null = null; -let mongoClient: MongoDBClient | null = null; +let container: ServiceContainer | null = null; +let app: Hono | null = null; // Initialize shutdown manager const shutdown = Shutdown.getInstance({ timeout: 15000 }); -// Add routes -app.route('/health', healthRoutes); -app.route('/api/exchanges', exchangeRoutes); - -// Basic API info endpoint -app.get('/', c => { - return c.json({ - name: 'Stock Bot Web API', - version: '1.0.0', - status: 'running', - timestamp: new Date().toISOString(), - endpoints: { - health: '/health', - exchanges: '/api/exchanges', - }, - }); -}); - -// Initialize services +// Initialize services with DI pattern async function initializeServices() { - logger.info('Initializing web API service...'); + logger.info('Initializing web API service with DI...'); try { - // Initialize MongoDB client - logger.debug('Connecting to MongoDB...'); - const mongoConfig = databaseConfig.mongodb; - mongoClient = new MongoDBClient( - { - uri: mongoConfig.uri, - database: mongoConfig.database, - host: mongoConfig.host, - port: mongoConfig.port, - timeouts: { - connectTimeout: 30000, - socketTimeout: 30000, - serverSelectionTimeout: 5000, - }, + // Create Awilix container with proper config structure + logger.debug('Creating Awilix DI container...'); + const awilixConfig = { + redis: { + host: config.database.dragonfly.host, + port: config.database.dragonfly.port, + db: config.database.dragonfly.db, }, - logger - ); - await mongoClient.connect(); - setMongoDBClient(mongoClient); - logger.info('MongoDB connected'); - - // Initialize PostgreSQL client - logger.debug('Connecting to PostgreSQL...'); - const pgConfig = databaseConfig.postgres; - postgresClient = new PostgreSQLClient( - { - host: pgConfig.host, - port: pgConfig.port, - database: pgConfig.database, - username: pgConfig.user, - password: pgConfig.password, - poolSettings: { - min: 2, - max: pgConfig.poolSize || 10, - idleTimeoutMillis: pgConfig.idleTimeout || 30000, - }, + mongodb: { + uri: config.database.mongodb.uri, + database: config.database.mongodb.database, }, - logger + postgres: { + host: config.database.postgres.host, + port: config.database.postgres.port, + database: config.database.postgres.database, + user: config.database.postgres.user, + password: config.database.postgres.password, + }, + questdb: { + enabled: false, // Web API doesn't need QuestDB + host: config.database.questdb.host, + httpPort: config.database.questdb.httpPort, + pgPort: config.database.questdb.pgPort, + influxPort: config.database.questdb.ilpPort, + database: config.database.questdb.database, + }, + }; + + container = createServiceContainer(awilixConfig); + await initializeAwilixServices(container); + logger.info('Awilix container created and initialized'); + + // Setup service-specific configuration + const serviceContainer = setupServiceContainer(config, container.resolve('serviceContainer')); + + // Initialize migration helper for backward compatibility + const { setContainerForMigration } = await import('./migration-helper'); + setContainerForMigration(serviceContainer); + logger.info('Migration helper initialized for backward compatibility'); + + // Create app with routes + app = new Hono(); + + // Add CORS middleware + app.use( + '*', + cors({ + origin: ['http://localhost:4200', 'http://localhost:3000', 'http://localhost:3002'], + allowMethods: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'], + allowHeaders: ['Content-Type', 'Authorization'], + credentials: true, + }) ); - await postgresClient.connect(); - setPostgreSQLClient(postgresClient); - logger.info('PostgreSQL connected'); + + // Basic API info endpoint + app.get('/', c => { + return c.json({ + name: 'Stock Bot Web API', + version: '1.0.0', + status: 'running', + timestamp: new Date().toISOString(), + endpoints: { + health: '/health', + exchanges: '/api/exchanges', + }, + }); + }); + + // Create and mount routes using the service container + const routes = createRoutes(serviceContainer); + app.route('/', routes); logger.info('All services initialized successfully'); } catch (error) { - logger.error('Failed to initialize services', { error }); + console.error('DETAILED ERROR:', error); + logger.error('Failed to initialize services', { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + details: JSON.stringify(error, null, 2) + }); throw error; } } @@ -128,17 +140,22 @@ async function initializeServices() { async function startServer() { await initializeServices(); + if (!app) { + throw new Error('App not initialized'); + } + server = Bun.serve({ port: PORT, fetch: app.fetch, development: config.environment === 'development', }); - logger.info(`Stock Bot Web API started on port ${PORT}`); + logger.info(`Web API service started on port ${PORT}`); } -// Register shutdown handlers -shutdown.onShutdown(async () => { +// Register shutdown handlers with priorities +// Priority 1: HTTP Server (high priority) +shutdown.onShutdownHigh(async () => { if (server) { logger.info('Stopping HTTP server...'); try { @@ -148,36 +165,42 @@ shutdown.onShutdown(async () => { logger.error('Error stopping HTTP server', { error }); } } -}); +}, 'HTTP Server'); -shutdown.onShutdown(async () => { - logger.info('Disconnecting from databases...'); +// Priority 2: Services and connections (medium priority) +shutdown.onShutdownMedium(async () => { + logger.info('Disposing services and connections...'); try { - if (mongoClient) { - await mongoClient.disconnect(); + if (container) { + // Disconnect database clients + const mongoClient = container.resolve('mongoClient'); + if (mongoClient?.disconnect) await mongoClient.disconnect(); + + const postgresClient = container.resolve('postgresClient'); + if (postgresClient?.disconnect) await postgresClient.disconnect(); + + logger.info('All services disposed successfully'); } - if (postgresClient) { - await postgresClient.disconnect(); - } - logger.info('Database connections closed'); } catch (error) { - logger.error('Error closing database connections', { error }); + logger.error('Error disposing services', { error }); } -}); +}, 'Services'); -shutdown.onShutdown(async () => { +// Priority 3: Logger shutdown (lowest priority - runs last) +shutdown.onShutdownLow(async () => { try { + logger.info('Shutting down loggers...'); await shutdownLoggers(); - // process.stdout.write('Web API loggers shut down\n'); - } catch (error) { - process.stderr.write(`Error shutting down loggers: ${error}\n`); + // Don't log after shutdown + } catch { + // Silently ignore logger shutdown errors } -}); +}, 'Loggers'); // Start the service startServer().catch(error => { - logger.error('Failed to start web API service', { error }); + logger.fatal('Failed to start web API service', { error }); process.exit(1); }); -logger.info('Web API service startup initiated'); +logger.info('Web API service startup initiated with DI pattern'); \ No newline at end of file diff --git a/apps/web-api/src/migration-helper.ts b/apps/web-api/src/migration-helper.ts new file mode 100644 index 0000000..349f050 --- /dev/null +++ b/apps/web-api/src/migration-helper.ts @@ -0,0 +1,30 @@ +/** + * Temporary migration helper for web-api service + * Provides backward compatibility while migrating to DI container + * + * TODO: Remove this file once all routes and services are migrated to use ServiceContainer + */ + +import type { ServiceContainer } from '@stock-bot/di'; +import type { MongoDBClient } from '@stock-bot/mongodb'; +import type { PostgreSQLClient } from '@stock-bot/postgres'; + +let containerInstance: ServiceContainer | null = null; + +export function setContainerForMigration(container: ServiceContainer): void { + containerInstance = container; +} + +export function getMongoDBClient(): MongoDBClient { + if (!containerInstance) { + throw new Error('Container not initialized. This is a migration helper - please update the service to accept ServiceContainer parameter'); + } + return containerInstance.mongodb; +} + +export function getPostgreSQLClient(): PostgreSQLClient { + if (!containerInstance) { + throw new Error('Container not initialized. This is a migration helper - please update the service to accept ServiceContainer parameter'); + } + return containerInstance.postgres; +} \ No newline at end of file diff --git a/apps/web-api/src/routes/create-routes.ts b/apps/web-api/src/routes/create-routes.ts new file mode 100644 index 0000000..11867ca --- /dev/null +++ b/apps/web-api/src/routes/create-routes.ts @@ -0,0 +1,24 @@ +/** + * Route factory for web API service + * Creates routes with access to the service container + */ + +import { Hono } from 'hono'; +import type { ServiceContainer } from '@stock-bot/di'; +import { healthRoutes, exchangeRoutes } from './index'; + +export function createRoutes(container: ServiceContainer): Hono { + const app = new Hono(); + + // Add container to context for all routes + app.use('*', async (c, next) => { + c.set('container', container); + await next(); + }); + + // Mount routes + app.route('/health', healthRoutes); + app.route('/api/exchanges', exchangeRoutes); + + return app; +} \ No newline at end of file