tests
This commit is contained in:
parent
3a7254708e
commit
b63e58784c
41 changed files with 5762 additions and 4477 deletions
|
|
@ -1,167 +1,166 @@
|
|||
import type { Page } from 'playwright';
|
||||
import type { BrowserOptions, ScrapingResult } from './types';
|
||||
|
||||
/**
|
||||
* Simple browser implementation for testing
|
||||
*/
|
||||
export class SimpleBrowser {
|
||||
private browser: any;
|
||||
private contexts = new Map<string, any>();
|
||||
private logger: any;
|
||||
private initialized = false;
|
||||
private _options: BrowserOptions = {
|
||||
headless: true,
|
||||
timeout: 30000,
|
||||
blockResources: false,
|
||||
enableNetworkLogging: false,
|
||||
};
|
||||
|
||||
constructor(logger?: any) {
|
||||
this.logger = logger || console;
|
||||
|
||||
// Initialize mock browser
|
||||
this.browser = {
|
||||
newContext: async () => {
|
||||
const pages: any[] = [];
|
||||
const context = {
|
||||
newPage: async () => {
|
||||
const page = {
|
||||
goto: async () => {},
|
||||
close: async () => {},
|
||||
evaluate: async () => {},
|
||||
waitForSelector: async () => {},
|
||||
screenshot: async () => Buffer.from('screenshot'),
|
||||
setViewport: async () => {},
|
||||
content: async () => '<html></html>',
|
||||
on: () => {},
|
||||
route: async () => {},
|
||||
};
|
||||
pages.push(page);
|
||||
return page;
|
||||
},
|
||||
close: async () => {},
|
||||
pages: async () => pages,
|
||||
};
|
||||
return context;
|
||||
},
|
||||
close: async () => {},
|
||||
isConnected: () => true,
|
||||
};
|
||||
}
|
||||
|
||||
async initialize(options: BrowserOptions = {}): Promise<void> {
|
||||
if (this.initialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Merge options
|
||||
this._options = { ...this._options, ...options };
|
||||
|
||||
this.logger.info('Initializing browser...');
|
||||
|
||||
// Mock browser is already initialized in constructor for simplicity
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
async createContext(id?: string): Promise<string> {
|
||||
if (!this.browser) {
|
||||
await this.initialize();
|
||||
}
|
||||
|
||||
const contextId = id || `context-${Date.now()}`;
|
||||
const context = await this.browser.newContext();
|
||||
this.contexts.set(contextId, context);
|
||||
return contextId;
|
||||
}
|
||||
|
||||
async closeContext(contextId: string): Promise<void> {
|
||||
const context = this.contexts.get(contextId);
|
||||
if (context) {
|
||||
await context.close();
|
||||
this.contexts.delete(contextId);
|
||||
}
|
||||
}
|
||||
|
||||
async newPage(contextId: string): Promise<Page> {
|
||||
const context = this.contexts.get(contextId);
|
||||
if (!context) {
|
||||
throw new Error(`Context ${contextId} not found`);
|
||||
}
|
||||
|
||||
const page = await context.newPage();
|
||||
|
||||
// Add resource blocking if enabled
|
||||
if (this._options?.blockResources) {
|
||||
await page.route('**/*.{png,jpg,jpeg,gif,svg,ico,woff,woff2,ttf,css}', (route: any) => {
|
||||
route.abort();
|
||||
});
|
||||
}
|
||||
|
||||
return page;
|
||||
}
|
||||
|
||||
async goto(page: Page, url: string, options?: any): Promise<void> {
|
||||
await page.goto(url, {
|
||||
timeout: this._options?.timeout || 30000,
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
async scrape(url: string, options?: { contextId?: string }): Promise<ScrapingResult> {
|
||||
try {
|
||||
let contextId = options?.contextId;
|
||||
const shouldCloseContext = !contextId;
|
||||
|
||||
if (!contextId) {
|
||||
contextId = await this.createContext();
|
||||
}
|
||||
|
||||
const page = await this.newPage(contextId);
|
||||
|
||||
await this.goto(page, url);
|
||||
|
||||
// Mock data for testing
|
||||
const data = {
|
||||
title: 'Test Title',
|
||||
text: 'Test content',
|
||||
links: ['link1', 'link2'],
|
||||
};
|
||||
|
||||
await page.close();
|
||||
|
||||
if (shouldCloseContext) {
|
||||
await this.closeContext(contextId);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data,
|
||||
url,
|
||||
};
|
||||
} catch (error: any) {
|
||||
return {
|
||||
success: false,
|
||||
error: error.message,
|
||||
url,
|
||||
data: {} as any,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (!this.browser) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Close all contexts
|
||||
for (const [contextId, context] of this.contexts) {
|
||||
await context.close();
|
||||
}
|
||||
this.contexts.clear();
|
||||
|
||||
await this.browser.close();
|
||||
this.browser = null;
|
||||
this.initialized = false;
|
||||
}
|
||||
|
||||
}
|
||||
import type { Page } from 'playwright';
|
||||
import type { BrowserOptions, ScrapingResult } from './types';
|
||||
|
||||
/**
|
||||
* Simple browser implementation for testing
|
||||
*/
|
||||
export class SimpleBrowser {
|
||||
private browser: any;
|
||||
private contexts = new Map<string, any>();
|
||||
private logger: any;
|
||||
private initialized = false;
|
||||
private _options: BrowserOptions = {
|
||||
headless: true,
|
||||
timeout: 30000,
|
||||
blockResources: false,
|
||||
enableNetworkLogging: false,
|
||||
};
|
||||
|
||||
constructor(logger?: any) {
|
||||
this.logger = logger || console;
|
||||
|
||||
// Initialize mock browser
|
||||
this.browser = {
|
||||
newContext: async () => {
|
||||
const pages: any[] = [];
|
||||
const context = {
|
||||
newPage: async () => {
|
||||
const page = {
|
||||
goto: async () => {},
|
||||
close: async () => {},
|
||||
evaluate: async () => {},
|
||||
waitForSelector: async () => {},
|
||||
screenshot: async () => Buffer.from('screenshot'),
|
||||
setViewport: async () => {},
|
||||
content: async () => '<html></html>',
|
||||
on: () => {},
|
||||
route: async () => {},
|
||||
};
|
||||
pages.push(page);
|
||||
return page;
|
||||
},
|
||||
close: async () => {},
|
||||
pages: async () => pages,
|
||||
};
|
||||
return context;
|
||||
},
|
||||
close: async () => {},
|
||||
isConnected: () => true,
|
||||
};
|
||||
}
|
||||
|
||||
async initialize(options: BrowserOptions = {}): Promise<void> {
|
||||
if (this.initialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Merge options
|
||||
this._options = { ...this._options, ...options };
|
||||
|
||||
this.logger.info('Initializing browser...');
|
||||
|
||||
// Mock browser is already initialized in constructor for simplicity
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
async createContext(id?: string): Promise<string> {
|
||||
if (!this.browser) {
|
||||
await this.initialize();
|
||||
}
|
||||
|
||||
const contextId = id || `context-${Date.now()}`;
|
||||
const context = await this.browser.newContext();
|
||||
this.contexts.set(contextId, context);
|
||||
return contextId;
|
||||
}
|
||||
|
||||
async closeContext(contextId: string): Promise<void> {
|
||||
const context = this.contexts.get(contextId);
|
||||
if (context) {
|
||||
await context.close();
|
||||
this.contexts.delete(contextId);
|
||||
}
|
||||
}
|
||||
|
||||
async newPage(contextId: string): Promise<Page> {
|
||||
const context = this.contexts.get(contextId);
|
||||
if (!context) {
|
||||
throw new Error(`Context ${contextId} not found`);
|
||||
}
|
||||
|
||||
const page = await context.newPage();
|
||||
|
||||
// Add resource blocking if enabled
|
||||
if (this._options?.blockResources) {
|
||||
await page.route('**/*.{png,jpg,jpeg,gif,svg,ico,woff,woff2,ttf,css}', (route: any) => {
|
||||
route.abort();
|
||||
});
|
||||
}
|
||||
|
||||
return page;
|
||||
}
|
||||
|
||||
async goto(page: Page, url: string, options?: any): Promise<void> {
|
||||
await page.goto(url, {
|
||||
timeout: this._options?.timeout || 30000,
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
async scrape(url: string, options?: { contextId?: string }): Promise<ScrapingResult> {
|
||||
try {
|
||||
let contextId = options?.contextId;
|
||||
const shouldCloseContext = !contextId;
|
||||
|
||||
if (!contextId) {
|
||||
contextId = await this.createContext();
|
||||
}
|
||||
|
||||
const page = await this.newPage(contextId);
|
||||
|
||||
await this.goto(page, url);
|
||||
|
||||
// Mock data for testing
|
||||
const data = {
|
||||
title: 'Test Title',
|
||||
text: 'Test content',
|
||||
links: ['link1', 'link2'],
|
||||
};
|
||||
|
||||
await page.close();
|
||||
|
||||
if (shouldCloseContext) {
|
||||
await this.closeContext(contextId);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data,
|
||||
url,
|
||||
};
|
||||
} catch (error: any) {
|
||||
return {
|
||||
success: false,
|
||||
error: error.message,
|
||||
url,
|
||||
data: {} as any,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (!this.browser) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Close all contexts
|
||||
for (const [_contextId, context] of this.contexts) {
|
||||
await context.close();
|
||||
}
|
||||
this.contexts.clear();
|
||||
|
||||
await this.browser.close();
|
||||
this.browser = null;
|
||||
this.initialized = false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue