integrated data-ingestion

apps/data-ingestion/config/default.json

@@ -12,6 +12,57 @@
       "credentials": false
     }
   },
+  "log": {
+    "level": "info",
+    "format": "json",
+    "hideObject": false,
+    "loki": {
+      "enabled": false,
+      "host": "localhost",
+      "port": 3100,
+      "labels": {}
+    }
+  },
+  "database": {
+    "postgres": {
+      "host": "localhost",
+      "port": 5432,
+      "database": "trading_bot",
+      "user": "trading_user",
+      "password": "trading_pass_dev",
+      "ssl": false,
+      "poolSize": 20,
+      "connectionTimeout": 30000,
+      "idleTimeout": 10000
+    },
+    "questdb": {
+      "host": "localhost",
+      "ilpPort": 9009,
+      "httpPort": 9000,
+      "pgPort": 8812,
+      "database": "questdb",
+      "user": "admin",
+      "password": "quest",
+      "bufferSize": 65536,
+      "flushInterval": 1000
+    },
+    "mongodb": {
+      "host": "localhost",
+      "port": 27017,
+      "database": "stock",
+      "user": "trading_admin",
+      "password": "trading_mongo_dev",
+      "authSource": "admin",
+      "poolSize": 20
+    },
+    "dragonfly": {
+      "host": "localhost",
+      "port": 6379,
+      "db": 0,
+      "maxRetries": 3,
+      "retryDelay": 100
+    }
+  },
   "queue": {
     "redis": {
       "host": "localhost",
@@ -24,12 +75,23 @@
         "type": "exponential",
         "delay": 1000
       },
-      "removeOnComplete": true,
-      "removeOnFail": false
+      "removeOnComplete": 100,
+      "removeOnFail": 50
     }
   },
   "webshare": {
     "apiKey": "",
     "apiUrl": "https://proxy.webshare.io/api/v2/"
+  },
+  "http": {
+    "timeout": 30000,
+    "retries": 3,
+    "retryDelay": 1000,
+    "userAgent": "StockBot/1.0",
+    "rateLimit": {
+      "enabled": false,
+      "requestsPerSecond": 10,
+      "burstSize": 20
+    }
   }
 }