From f70a8be1cb826d2598237235826152cfb5873983 Mon Sep 17 00:00:00 2001 From: Bojan Kucera Date: Sat, 7 Jun 2025 12:57:55 -0400 Subject: [PATCH] simplified httpclient lib --- bun.lock | 59 ++++- libs/http-client/package.json | 4 +- libs/http-client/src/client.ts | 331 ++++++++++++-------------- libs/http-client/src/proxy-manager.ts | 63 +++-- libs/http-client/src/types.ts | 1 - libs/logger/package.json | 1 + 6 files changed, 256 insertions(+), 203 deletions(-) diff --git a/bun.lock b/bun.lock index cf4f5df..e8e81a1 100644 --- a/bun.lock +++ b/bun.lock @@ -89,10 +89,10 @@ "dependencies": { "@stock-bot/logger": "*", "@stock-bot/types": "*", + "got": "^14.4.2", "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", "socks-proxy-agent": "^8.0.5", - "undici": "^7.10.0", }, "devDependencies": { "@types/node": "^20.11.0", @@ -108,6 +108,7 @@ "version": "1.0.0", "dependencies": { "@stock-bot/config": "*", + "got": "^14.4.7", "pino": "^9.7.0", "pino-loki": "^2.6.0", "pino-pretty": "^13.0.0", @@ -281,6 +282,10 @@ "@protobufjs/utf8": ["@protobufjs/utf8@1.1.0", "", {}, "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="], + "@sec-ant/readable-stream": ["@sec-ant/readable-stream@0.4.1", "", {}, "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg=="], + + "@sindresorhus/is": ["@sindresorhus/is@7.0.2", "", {}, "sha512-d9xRovfKNz1SKieM0qJdO+PQonjnnIfSNWfHYnBSJ9hkjm0ZPw6HlxscDXYstp3z+7V2GOFHc+J0CYrYTjqCJw=="], + "@stock-bot/cache": ["@stock-bot/cache@workspace:libs/cache"], "@stock-bot/config": ["@stock-bot/config@workspace:libs/config"], @@ -307,6 +312,8 @@ "@stock-bot/vector-engine": ["@stock-bot/vector-engine@workspace:libs/vector-engine"], + "@szmarczak/http-timer": ["@szmarczak/http-timer@5.0.1", "", { "dependencies": { "defer-to-connect": "^2.0.1" } }, "sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw=="], + "@testcontainers/mongodb": ["@testcontainers/mongodb@10.28.0", "", { "dependencies": { "testcontainers": "^10.28.0" } }, "sha512-78h6n2jnFOQ8IfPjgL1+vsHuEeA0itclEOpx9kkQR+FOWnwJN9AeeX6+rMmZCtRgTsr5wT0BvfFoDssMkDqWaQ=="], "@testcontainers/postgresql": ["@testcontainers/postgresql@10.28.0", "", { "dependencies": { "testcontainers": "^10.28.0" } }, "sha512-NN25rruG5D4Q7pCNIJuHwB+G85OSeJ3xHZ2fWx0O6sPoPEfCYwvpj8mq99cyn68nxFkFYZeyrZJtSFO+FnydiA=="], @@ -319,6 +326,8 @@ "@types/dockerode": ["@types/dockerode@3.3.40", "", { "dependencies": { "@types/docker-modem": "*", "@types/node": "*", "@types/ssh2": "*" } }, "sha512-O1ckSFYbcYv/KcnAHMLCnKQYY8/5+6CRzpsOPcQIePHRX2jG4Gmz8uXPMCXIxTGN9OYkE5eox/L67l2sGY1UYg=="], + "@types/http-cache-semantics": ["@types/http-cache-semantics@4.0.4", "", {}, "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA=="], + "@types/json-schema": ["@types/json-schema@7.0.15", "", {}, "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA=="], "@types/methods": ["@types/methods@1.1.4", "", {}, "sha512-ymXWVrDiCxTBE3+RIrrP533E70eA+9qu7zdWoHuOmGujkYtzf4HQF96b8nwHLqhuf4ykX61IGRIB38CC6/sImQ=="], @@ -431,6 +440,10 @@ "byline": ["byline@5.0.0", "", {}, "sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q=="], + "cacheable-lookup": ["cacheable-lookup@7.0.0", "", {}, "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w=="], + + "cacheable-request": ["cacheable-request@12.0.1", "", { "dependencies": { "@types/http-cache-semantics": "^4.0.4", "get-stream": "^9.0.1", "http-cache-semantics": "^4.1.1", "keyv": "^4.5.4", "mimic-response": "^4.0.0", "normalize-url": "^8.0.1", "responselike": "^3.0.0" } }, "sha512-Yo9wGIQUaAfIbk+qY0X4cDQgCosecfBe3V9NSyeY4qPC2SAkbCS4Xj79VP8WOzitpJUZKc/wsRCYF5ariDIwkg=="], + "call-bind": ["call-bind@1.0.8", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.0", "es-define-property": "^1.0.0", "get-intrinsic": "^1.2.4", "set-function-length": "^1.2.2" } }, "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww=="], "call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="], @@ -485,8 +498,12 @@ "debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="], + "decompress-response": ["decompress-response@6.0.0", "", { "dependencies": { "mimic-response": "^3.1.0" } }, "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ=="], + "deep-is": ["deep-is@0.1.4", "", {}, "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ=="], + "defer-to-connect": ["defer-to-connect@2.0.1", "", {}, "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg=="], + "define-data-property": ["define-data-property@1.1.4", "", { "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", "gopd": "^1.0.1" } }, "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A=="], "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="], @@ -589,6 +606,8 @@ "form-data": ["form-data@4.0.3", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-qsITQPfmvMOSAdeyZ+12I1c+CKSstAFAwu+97zrnWAbIr5u8wfsExUzCesVLC8NgHuRUqNN4Zy6UPWUTRGslcA=="], + "form-data-encoder": ["form-data-encoder@4.1.0", "", {}, "sha512-G6NsmEW15s0Uw9XnCg+33H3ViYRyiM0hMrMhhqQOR8NFc5GhYrI+6I3u7OTw7b91J2g8rtvMBZJDbcGb2YUniw=="], + "formidable": ["formidable@2.1.5", "", { "dependencies": { "@paralleldrive/cuid2": "^2.2.2", "dezalgo": "^1.0.4", "once": "^1.4.0", "qs": "^6.11.0" } }, "sha512-Oz5Hwvwak/DCaXVVUtPn4oLMLLy1CdclLKO1LFgU7XzDpVMUU5UjlSLpGMocyQNNk8F6IJW9M/YdooSn2MRI+Q=="], "fs-constants": ["fs-constants@1.0.0", "", {}, "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="], @@ -607,6 +626,8 @@ "get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="], + "get-stream": ["get-stream@9.0.1", "", { "dependencies": { "@sec-ant/readable-stream": "^0.4.1", "is-stream": "^4.0.1" } }, "sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA=="], + "glob": ["glob@10.4.5", "", { "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", "minimatch": "^9.0.4", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", "path-scurry": "^1.11.1" }, "bin": { "glob": "dist/esm/bin.mjs" } }, "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg=="], "glob-parent": ["glob-parent@6.0.2", "", { "dependencies": { "is-glob": "^4.0.3" } }, "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A=="], @@ -617,6 +638,8 @@ "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], + "got": ["got@14.4.7", "", { "dependencies": { "@sindresorhus/is": "^7.0.1", "@szmarczak/http-timer": "^5.0.1", "cacheable-lookup": "^7.0.0", "cacheable-request": "^12.0.1", "decompress-response": "^6.0.0", "form-data-encoder": "^4.0.2", "http2-wrapper": "^2.2.1", "lowercase-keys": "^3.0.0", "p-cancelable": "^4.0.1", "responselike": "^3.0.0", "type-fest": "^4.26.1" } }, "sha512-DI8zV1231tqiGzOiOzQWDhsBmncFW7oQDH6Zgy6pDPrqJuVZMtoSgPLLsBZQj8Jg4JFfwoOsDA8NGtLQLnIx2g=="], + "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], "graphemer": ["graphemer@1.4.0", "", {}, "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag=="], @@ -633,8 +656,12 @@ "help-me": ["help-me@5.0.0", "", {}, "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg=="], + "http-cache-semantics": ["http-cache-semantics@4.2.0", "", {}, "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ=="], + "http-proxy-agent": ["http-proxy-agent@7.0.2", "", { "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" } }, "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig=="], + "http2-wrapper": ["http2-wrapper@2.2.1", "", { "dependencies": { "quick-lru": "^5.1.1", "resolve-alpn": "^1.2.0" } }, "sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ=="], + "https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="], "ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="], @@ -665,7 +692,7 @@ "is-path-inside": ["is-path-inside@3.0.3", "", {}, "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ=="], - "is-stream": ["is-stream@2.0.1", "", {}, "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="], + "is-stream": ["is-stream@4.0.1", "", {}, "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A=="], "isarray": ["isarray@2.0.5", "", {}, "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw=="], @@ -709,6 +736,8 @@ "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], + "lowercase-keys": ["lowercase-keys@3.0.0", "", {}, "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ=="], + "lru-cache": ["lru-cache@6.0.0", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA=="], "make-dir": ["make-dir@3.1.0", "", { "dependencies": { "semver": "^6.0.0" } }, "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw=="], @@ -729,6 +758,8 @@ "mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], + "mimic-response": ["mimic-response@4.0.0", "", {}, "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg=="], + "minimatch": ["minimatch@3.1.2", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw=="], "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], @@ -763,6 +794,8 @@ "normalize-path": ["normalize-path@3.0.0", "", {}, "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="], + "normalize-url": ["normalize-url@8.0.1", "", {}, "sha512-IO9QvjUMWxPQQhs60oOu10CRkWCiZzSUkzbXGGV9pviYl1fXYcvkzQ5jV9z8Y6un8ARoVRl4EtC6v6jNqbaJ/w=="], + "object-hash": ["object-hash@2.2.0", "", {}, "sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw=="], "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], @@ -775,6 +808,8 @@ "optionator": ["optionator@0.9.4", "", { "dependencies": { "deep-is": "^0.1.3", "fast-levenshtein": "^2.0.6", "levn": "^0.4.1", "prelude-ls": "^1.2.1", "type-check": "^0.4.0", "word-wrap": "^1.2.5" } }, "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g=="], + "p-cancelable": ["p-cancelable@4.0.1", "", {}, "sha512-wBowNApzd45EIKdO1LaU+LrMBwAcjfPaYtVzV3lmfM3gf8Z4CHZsiIqlM8TZZ8okYvh5A1cP6gTfCRQtwUpaUg=="], + "p-limit": ["p-limit@3.1.0", "", { "dependencies": { "yocto-queue": "^0.1.0" } }, "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ=="], "p-locate": ["p-locate@5.0.0", "", { "dependencies": { "p-limit": "^3.0.2" } }, "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw=="], @@ -863,6 +898,8 @@ "quick-format-unescaped": ["quick-format-unescaped@4.0.4", "", {}, "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg=="], + "quick-lru": ["quick-lru@5.1.1", "", {}, "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA=="], + "railroad-diagrams": ["railroad-diagrams@1.0.0", "", {}, "sha512-cz93DjNeLY0idrCNOH6PviZGRN9GJhsdm9hpn1YCS879fj4W+x5IFJhhkRZcwVgMmFF7R82UA/7Oh+R8lLZg6A=="], "randexp": ["randexp@0.4.6", "", { "dependencies": { "discontinuous-range": "1.0.0", "ret": "~0.1.10" } }, "sha512-80WNmd9DA0tmZrw9qQa62GPPWfuXJknrmVmLcxvq4uZBdYqb1wYoKTmnlGUchvVWe0XiLupYkBoXVOxz3C8DYQ=="], @@ -879,8 +916,12 @@ "require-directory": ["require-directory@2.1.1", "", {}, "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q=="], + "resolve-alpn": ["resolve-alpn@1.2.1", "", {}, "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g=="], + "resolve-from": ["resolve-from@4.0.0", "", {}, "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g=="], + "responselike": ["responselike@3.0.0", "", { "dependencies": { "lowercase-keys": "^3.0.0" } }, "sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg=="], + "ret": ["ret@0.1.15", "", {}, "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg=="], "retry": ["retry@0.12.0", "", {}, "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow=="], @@ -1001,11 +1042,11 @@ "type-check": ["type-check@0.4.0", "", { "dependencies": { "prelude-ls": "^1.2.1" } }, "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew=="], - "type-fest": ["type-fest@0.20.2", "", {}, "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ=="], + "type-fest": ["type-fest@4.41.0", "", {}, "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA=="], "typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="], - "undici": ["undici@7.10.0", "", {}, "sha512-u5otvFBOBZvmdjWLVW+5DAc9Nkq8f24g0O9oY7qw2JVIF1VocIFoyz9JFkuVOS2j41AufeO0xnlweJ2RLT8nGw=="], + "undici": ["undici@5.29.0", "", { "dependencies": { "@fastify/busboy": "^2.0.0" } }, "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg=="], "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], @@ -1059,10 +1100,16 @@ "@typescript-eslint/typescript-estree/minimatch": ["minimatch@9.0.3", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg=="], + "archiver-utils/is-stream": ["is-stream@2.0.1", "", {}, "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="], + "bl/buffer": ["buffer@5.7.1", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" } }, "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ=="], "bl/readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], + "compress-commons/is-stream": ["is-stream@2.0.1", "", {}, "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="], + + "decompress-response/mimic-response": ["mimic-response@3.1.0", "", {}, "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ=="], + "docker-modem/readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], "dockerode/tar-fs": ["tar-fs@2.1.3", "", { "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", "tar-stream": "^2.1.4" } }, "sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg=="], @@ -1075,6 +1122,8 @@ "glob/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="], + "globals/type-fest": ["type-fest@0.20.2", "", {}, "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ=="], + "lazystream/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], "make-dir/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], @@ -1091,8 +1140,6 @@ "rimraf/glob": ["glob@7.2.3", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q=="], - "testcontainers/undici": ["undici@5.29.0", "", { "dependencies": { "@fastify/busboy": "^2.0.0" } }, "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg=="], - "yauzl/buffer-crc32": ["buffer-crc32@0.2.13", "", {}, "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ=="], "@isaacs/cliui/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="], diff --git a/libs/http-client/package.json b/libs/http-client/package.json index ed739af..b1e8f79 100644 --- a/libs/http-client/package.json +++ b/libs/http-client/package.json @@ -16,10 +16,10 @@ "dependencies": { "@stock-bot/logger": "*", "@stock-bot/types": "*", + "got": "^14.4.7", "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", - "socks-proxy-agent": "^8.0.5", - "undici": "^7.10.0" + "socks-proxy-agent": "^8.0.5" }, "devDependencies": { "@types/node": "^20.11.0", diff --git a/libs/http-client/src/client.ts b/libs/http-client/src/client.ts index cc0d0c1..45d671a 100644 --- a/libs/http-client/src/client.ts +++ b/libs/http-client/src/client.ts @@ -6,7 +6,7 @@ import type { } from './types.js'; import { HttpError } from './types.js'; import { ProxyManager } from './proxy-manager.js'; -import { request } from 'undici'; +import got from 'got'; export class HttpClient { private readonly config: HttpClientConfig; @@ -15,49 +15,9 @@ export class HttpClient { constructor(config: HttpClientConfig = {}, logger?: Logger) { this.config = config; this.logger = logger; - - // Validate proxy configuration if provided - if (this.config.proxy) { - ProxyManager.validateConfig(this.config.proxy); - } } - /** - * Make an HTTP request using hybrid approach: - * - Bun fetch for HTTP/HTTPS proxies (fastest ~150-300ms) - * - Undici for SOCKS proxies (fast ~300-600ms) - */ - async request(config: RequestConfig): Promise> { - const finalConfig = this.mergeConfig(config); - - this.logger?.debug('Making HTTP request', { - method: finalConfig.method, - url: finalConfig.url - }); - - try { - const response = await this.executeRequest(finalConfig); - - this.logger?.debug('HTTP request successful', { - method: finalConfig.method, - url: finalConfig.url, - status: response.status, - }); - - return response; - } catch (error) { - this.logger?.warn('HTTP request failed', { - method: finalConfig.method, - url: finalConfig.url, - error: (error as Error).message, - }); - throw error; - } - } - - /** - * Convenience methods for common HTTP methods - */ + // Convenience methods async get(url: string, config: Omit = {}): Promise> { return this.request({ ...config, method: 'GET', url }); } @@ -77,186 +37,209 @@ export class HttpClient { async patch(url: string, body?: any, config: Omit = {}): Promise> { return this.request({ ...config, method: 'PATCH', url, body }); } - /** - * Execute HTTP request using hybrid approach + * Main request method - unified and simplified */ - private async executeRequest(config: RequestConfig): Promise> { - const timeout = config.timeout ?? this.config.timeout ?? 30000; + async request(config: RequestConfig): Promise> { + const finalConfig = this.mergeConfig(config); - // Decide between Bun fetch and Undici based on proxy type - if (this.config.proxy) { - if (ProxyManager.shouldUseBunFetch(this.config.proxy)) { - return this.executeBunRequest(config, timeout); - } else { - return this.executeUndiciRequest(config, timeout); - } - } else { - // No proxy - use fast Bun fetch - return this.executeBunRequest(config, timeout); + this.logger?.debug('Making HTTP request', { + method: finalConfig.method, + url: finalConfig.url, + hasProxy: !!finalConfig.proxy + }); + + try { + // Single decision point for proxy type - only request-level proxy + const proxy = finalConfig.proxy; + const useBunFetch = !proxy || ProxyManager.shouldUseBunFetch(proxy); + + const response = useBunFetch + ? await this.fetchRequest(finalConfig) + : await this.gotRequest(finalConfig); + + this.logger?.debug('HTTP request successful', { + method: finalConfig.method, + url: finalConfig.url, + status: response.status, + }); + + return response; + } catch (error) { + this.logger?.warn('HTTP request failed', { + method: finalConfig.method, + url: finalConfig.url, + error: (error as Error).message, + }); + throw error; } } /** - * Execute request using Bun's native fetch (fastest for HTTP/HTTPS proxies) + * Bun fetch implementation (simplified) */ - private async executeBunRequest(config: RequestConfig, timeout: number): Promise> { - const abortController = new AbortController(); - const timeoutId = setTimeout(() => abortController.abort(), timeout); + private async fetchRequest(config: RequestConfig): Promise> { + const timeout = config.timeout ?? this.config.timeout ?? 30000; + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeout); try { - const requestOptions: RequestInit = { - method: config.method || 'GET', - headers: config.headers || {}, - signal: abortController.signal, - }; - - // Add body for non-GET requests - if (config.body && config.method !== 'GET') { - if (typeof config.body === 'object') { - requestOptions.body = JSON.stringify(config.body); - requestOptions.headers = { - 'Content-Type': 'application/json', - ...requestOptions.headers, - }; - } else { - requestOptions.body = config.body; - } - } - - // Add proxy for Bun fetch (HTTP/HTTPS only) - if (this.config.proxy && ProxyManager.shouldUseBunFetch(this.config.proxy)) { - const proxyUrl = ProxyManager.createBunProxyUrl(this.config.proxy); - // Bun supports proxy via environment or direct configuration - (requestOptions as any).proxy = proxyUrl; - } - - const response = await fetch(config.url, requestOptions); + const options = this.buildFetchOptions(config, controller.signal); + const response = await fetch(config.url, options); + clearTimeout(timeoutId); - - return await this.parseResponse(response); + return this.parseFetchResponse(response); } catch (error) { clearTimeout(timeoutId); - if (abortController.signal.aborted) { - throw new HttpError(`Request timeout after ${timeout}ms`); - } + throw controller.signal.aborted + ? new HttpError(`Request timeout after ${timeout}ms`) + : new HttpError(`Request failed: ${(error as Error).message}`); + } + } + + /** + * Got implementation (simplified for SOCKS proxies) + */ + private async gotRequest(config: RequestConfig): Promise> { + const timeout = config.timeout ?? this.config.timeout ?? 30000; + + try { + const options = this.buildGotOptions(config, timeout); + const response = await got(config.url, options); + + return this.parseGotResponse(response); + } catch (error) { throw new HttpError(`Request failed: ${(error as Error).message}`); } } - /** - * Execute request using Undici (fast for SOCKS proxies) + * Build fetch options (extracted for clarity) */ - private async executeUndiciRequest(config: RequestConfig, timeout: number): Promise> { - try { - const requestOptions: any = { - method: config.method || 'GET', - headers: config.headers || {}, - headersTimeout: timeout, - bodyTimeout: timeout, - }; + private buildFetchOptions(config: RequestConfig, signal: AbortSignal): RequestInit { + const options: RequestInit = { + method: config.method || 'GET', + headers: config.headers || {}, + signal, + }; - // Add body for non-GET requests - if (config.body && config.method !== 'GET') { - if (typeof config.body === 'object') { - requestOptions.body = JSON.stringify(config.body); - requestOptions.headers = { - 'Content-Type': 'application/json', - ...requestOptions.headers, - }; - } else { - requestOptions.body = config.body; - } + // Add body + if (config.body && config.method !== 'GET') { + if (typeof config.body === 'object') { + options.body = JSON.stringify(config.body); + options.headers = { 'Content-Type': 'application/json', ...options.headers }; + } else { + options.body = config.body; } - - // Add SOCKS proxy via Undici agent - if (this.config.proxy && !ProxyManager.shouldUseBunFetch(this.config.proxy)) { - requestOptions.dispatcher = ProxyManager.createUndiciAgent(this.config.proxy); - } - console.log('Executing Undici request', requestOptions) - - const response = await request(config.url, requestOptions); - - // Convert Undici response to our format - const data = await this.parseUndiciResponse(response); - - return { - data, - status: response.statusCode, - headers: response.headers as Record, - ok: response.statusCode >= 200 && response.statusCode < 300, - }; - } catch (error) { - throw new HttpError(`Undici request failed: ${(error as Error).message}`); } + + // Add proxy (HTTP/HTTPS only) - request level only + if (config.proxy && ProxyManager.shouldUseBunFetch(config.proxy)) { + (options as any).proxy = ProxyManager.createBunProxyUrl(config.proxy); + } + + return options; + } + /** + * Build Got options (extracted for clarity) + */ + private buildGotOptions(config: RequestConfig, timeout: number): any { + const options: any = { + method: config.method || 'GET', + headers: config.headers || {}, + timeout: { + request: timeout, + connect: 10000 + }, + retry: { + limit: 3, + methods: ['GET', 'POST', 'PUT', 'DELETE', 'PATCH'] + }, + throwHttpErrors: false, + responseType: 'json' + }; + + // Add body + if (config.body && config.method !== 'GET') { + if (typeof config.body === 'object') { + options.json = config.body; + } else { + options.body = config.body; + options.headers = { 'Content-Type': 'text/plain', ...options.headers }; + } + } + + // Add SOCKS proxy via agent - request level only + if (config.proxy && !ProxyManager.shouldUseBunFetch(config.proxy)) { + ProxyManager.validateConfig(config.proxy); + const agent = ProxyManager.createGotAgent(config.proxy); + options.agent = { + http: agent, + https: agent + }; + } + + return options; } /** - * Parse standard fetch response + * Parse fetch response (simplified) */ - private async parseResponse(response: Response): Promise> { - const responseHeaders: Record = {}; - response.headers.forEach((value, key) => { - responseHeaders[key] = value; - }); - - let data: T; - const contentType = response.headers.get('content-type') || ''; - - if (contentType.includes('application/json')) { - data = await response.json(); - } else if (contentType.includes('text/')) { - data = await response.text() as any; - } else { - data = await response.arrayBuffer() as any; - } + private async parseFetchResponse(response: Response): Promise> { + const data = await this.parseResponseBody(response); + const headers = Object.fromEntries(response.headers.entries()); if (!response.ok) { throw new HttpError( `Request failed with status ${response.status}`, response.status, - { - data, - status: response.status, - headers: responseHeaders, - ok: response.ok, - } + { data, status: response.status, headers, ok: response.ok } ); } - return { - data, - status: response.status, - headers: responseHeaders, - ok: response.ok, - }; + return { data, status: response.status, headers, ok: response.ok }; } /** - * Parse Undici response + * Parse Got response (simplified) */ - private async parseUndiciResponse(response: any): Promise { - const contentType = response.headers['content-type'] || ''; + private parseGotResponse(response: any): HttpResponse { + const headers = response.headers as Record; + const status = response.statusCode; + const ok = status >= 200 && status < 300; + const data = response.body as T; + + if (!ok) { + throw new HttpError( + `Request failed with status ${status}`, + status, + { data, status, headers, ok } + ); + } + + return { data, status, headers, ok }; + } + + /** + * Unified body parsing (works for fetch response) + */ + private async parseResponseBody(response: Response): Promise { + const contentType = response.headers.get('content-type') || ''; if (contentType.includes('application/json')) { - return await response.body.json(); + return response.json(); } else if (contentType.includes('text/')) { - return await response.body.text(); + return response.text() as any; } else { - return await response.body.arrayBuffer(); + return response.arrayBuffer() as any; } } - /** - * Merge request config with default config + * Merge configs - request-level proxy only */ private mergeConfig(config: RequestConfig): RequestConfig { return { ...config, - headers: { - ...this.config.headers, - ...config.headers, - }, + headers: { ...this.config.headers, ...config.headers }, timeout: config.timeout ?? this.config.timeout, }; } diff --git a/libs/http-client/src/proxy-manager.ts b/libs/http-client/src/proxy-manager.ts index ce9a225..673badf 100644 --- a/libs/http-client/src/proxy-manager.ts +++ b/libs/http-client/src/proxy-manager.ts @@ -1,10 +1,12 @@ -import { ProxyAgent } from 'undici'; +import got from 'got'; import { SocksProxyAgent } from 'socks-proxy-agent'; +import { HttpsProxyAgent } from 'https-proxy-agent'; +import { HttpProxyAgent } from 'http-proxy-agent'; import type { ProxyConfig } from './types.js'; export class ProxyManager { /** - * Determine if we should use Bun fetch (HTTP/HTTPS) or Undici (SOCKS) + * Determine if we should use Bun fetch (HTTP/HTTPS) or Got (SOCKS) */ static shouldUseBunFetch(proxy: ProxyConfig): boolean { return proxy.protocol === 'http' || proxy.protocol === 'https'; @@ -23,35 +25,56 @@ export class ProxyManager { } /** - * Create agent for SOCKS proxies (used with undici) + * Create appropriate agent for Got based on proxy type */ - static createSocksAgent(proxy: ProxyConfig): SocksProxyAgent { - const { protocol, host, port, username, password } = proxy; + static createGotAgent(proxy: ProxyConfig) { + this.validateConfig(proxy); - let proxyUrl: string; - if (username && password) { - proxyUrl = `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`; - } else { - proxyUrl = `${protocol}://${host}:${port}`; - } + const proxyUrl = this.buildProxyUrl(proxy); - return new SocksProxyAgent(proxyUrl); + switch (proxy.protocol) { + case 'socks4': + case 'socks5': + return new SocksProxyAgent(proxyUrl); + case 'http': + return new HttpProxyAgent(proxyUrl); + case 'https': + return new HttpsProxyAgent(proxyUrl); + default: + throw new Error(`Unsupported proxy protocol: ${proxy.protocol}`); + } } /** - * Create Undici proxy agent for HTTP/HTTPS proxies (fallback) + * Create Got instance with proxy configuration */ - static createUndiciAgent(proxy: ProxyConfig): ProxyAgent { + static createGotInstance(proxy: ProxyConfig) { + const agent = this.createGotAgent(proxy); + + return got.extend({ + agent: { + http: agent, + https: agent + }, + timeout: { + request: 30000, + connect: 10000 + }, + retry: { + limit: 3, + methods: ['GET', 'POST', 'PUT', 'DELETE', 'PATCH'] + }, + throwHttpErrors: false // We'll handle errors ourselves + }); + } + + private static buildProxyUrl(proxy: ProxyConfig): string { const { protocol, host, port, username, password } = proxy; - let proxyUrl: string; if (username && password) { - proxyUrl = `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`; - } else { - proxyUrl = `${protocol}://${host}:${port}`; + return `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`; } - - return new ProxyAgent(proxyUrl); + return `${protocol}://${host}:${port}`; } /** diff --git a/libs/http-client/src/types.ts b/libs/http-client/src/types.ts index 5f4c020..331e75f 100644 --- a/libs/http-client/src/types.ts +++ b/libs/http-client/src/types.ts @@ -11,7 +11,6 @@ export interface ProxyConfig { export interface HttpClientConfig { timeout?: number; - proxy?: ProxyConfig; headers?: Record; } diff --git a/libs/logger/package.json b/libs/logger/package.json index 1cac230..39bcffa 100644 --- a/libs/logger/package.json +++ b/libs/logger/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@stock-bot/config": "*", + "got": "^14.4.7", "pino": "^9.7.0", "pino-loki": "^2.6.0", "pino-pretty": "^13.0.0"