removed junk
This commit is contained in:
parent
4ad232c35e
commit
042b8cb83a
1 changed files with 0 additions and 103 deletions
|
|
@ -1,103 +0,0 @@
|
|||
import { Page } from 'playwright';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { Browser } from './browser';
|
||||
import type { ScrapingResult } from './types';
|
||||
|
||||
interface TabInfo {
|
||||
page: Page;
|
||||
contextId: string;
|
||||
}
|
||||
|
||||
export class BrowserTabManager {
|
||||
private tabs: Map<string, TabInfo> = new Map();
|
||||
private logger = getLogger('browser-tab-manager');
|
||||
|
||||
async createTab(url?: string): Promise<{ page: Page; tabId: string }> {
|
||||
const tabId = `tab-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
||||
const { page, contextId } = await Browser.createPageWithProxy(url || 'about:blank');
|
||||
|
||||
this.tabs.set(tabId, { page, contextId });
|
||||
this.logger.debug('Tab created', { tabId, url });
|
||||
|
||||
return { page, tabId };
|
||||
}
|
||||
|
||||
async createTabWithProxy(
|
||||
url: string,
|
||||
proxy: string
|
||||
): Promise<{ page: Page; tabId: string; contextId: string }> {
|
||||
const tabId = `tab-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
||||
const { page, contextId } = await Browser.createPageWithProxy(url, proxy);
|
||||
|
||||
this.tabs.set(tabId, { page, contextId });
|
||||
this.logger.debug('Tab with proxy created', { tabId, url, proxy });
|
||||
|
||||
return { page, tabId, contextId };
|
||||
}
|
||||
|
||||
async scrapeUrlsWithProxies<T>(
|
||||
urlProxyPairs: Array<{ url: string; proxy: string }>,
|
||||
extractor: (page: Page) => Promise<T>,
|
||||
options: { concurrency?: number } = {}
|
||||
): Promise<ScrapingResult<T>[]> {
|
||||
const { concurrency = 3 } = options;
|
||||
const results: ScrapingResult<T>[] = [];
|
||||
|
||||
for (let i = 0; i < urlProxyPairs.length; i += concurrency) {
|
||||
const batch = urlProxyPairs.slice(i, i + concurrency);
|
||||
|
||||
const batchPromises = batch.map(async ({ url, proxy }) => {
|
||||
let tabId: string | undefined;
|
||||
|
||||
try {
|
||||
const result = await this.createTabWithProxy(url, proxy);
|
||||
tabId = result.tabId;
|
||||
|
||||
const data = await extractor(result.page);
|
||||
|
||||
return {
|
||||
data,
|
||||
url,
|
||||
success: true,
|
||||
} as ScrapingResult<T>;
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
|
||||
return {
|
||||
data: null as T,
|
||||
url,
|
||||
success: false,
|
||||
error: errorMessage,
|
||||
} as ScrapingResult<T>;
|
||||
} finally {
|
||||
if (tabId) {
|
||||
await this.closeTab(tabId);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const batchResults = await Promise.all(batchPromises);
|
||||
results.push(...batchResults);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async closeTab(tabId: string): Promise<void> {
|
||||
const tab = this.tabs.get(tabId);
|
||||
if (tab) {
|
||||
await tab.page.close();
|
||||
await Browser.closeContext(tab.contextId);
|
||||
this.tabs.delete(tabId);
|
||||
this.logger.debug('Tab closed', { tabId });
|
||||
}
|
||||
}
|
||||
|
||||
getTabCount(): number {
|
||||
return this.tabs.size;
|
||||
}
|
||||
|
||||
getAllTabIds(): string[] {
|
||||
return Array.from(this.tabs.keys());
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue