adding data-services
This commit is contained in:
parent
e3bfd05b90
commit
405b818c86
139 changed files with 55943 additions and 416 deletions
|
|
@ -0,0 +1,297 @@
|
|||
import { Context } from 'hono';
|
||||
import { logger } from '@stock-bot/utils';
|
||||
import { DataPipelineOrchestrator } from '../core/DataPipelineOrchestrator';
|
||||
import { JobStatus } from '../types/DataPipeline';
|
||||
|
||||
export class JobController {
|
||||
constructor(private orchestrator: DataPipelineOrchestrator) {}
|
||||
|
||||
async listJobs(c: Context): Promise<Response> {
|
||||
try {
|
||||
const pipelineId = c.req.query('pipelineId');
|
||||
const status = c.req.query('status') as JobStatus;
|
||||
const limit = parseInt(c.req.query('limit') || '50');
|
||||
const offset = parseInt(c.req.query('offset') || '0');
|
||||
|
||||
let jobs = this.orchestrator.listJobs(pipelineId);
|
||||
|
||||
// Filter by status if provided
|
||||
if (status) {
|
||||
jobs = jobs.filter(job => job.status === status);
|
||||
}
|
||||
|
||||
// Sort by creation time (newest first)
|
||||
jobs.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
|
||||
|
||||
// Apply pagination
|
||||
const totalJobs = jobs.length;
|
||||
const paginatedJobs = jobs.slice(offset, offset + limit);
|
||||
|
||||
return c.json({
|
||||
success: true,
|
||||
data: paginatedJobs,
|
||||
pagination: {
|
||||
total: totalJobs,
|
||||
limit,
|
||||
offset,
|
||||
hasMore: offset + limit < totalJobs
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to list jobs:', error);
|
||||
|
||||
return c.json({
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Failed to list jobs'
|
||||
}, 500);
|
||||
}
|
||||
}
|
||||
|
||||
async getJob(c: Context): Promise<Response> {
|
||||
try {
|
||||
const jobId = c.req.param('id');
|
||||
const job = this.orchestrator.getJob(jobId);
|
||||
|
||||
if (!job) {
|
||||
return c.json({
|
||||
success: false,
|
||||
error: 'Job not found'
|
||||
}, 404);
|
||||
}
|
||||
|
||||
return c.json({
|
||||
success: true,
|
||||
data: job
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get job:', error);
|
||||
|
||||
return c.json({
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Failed to get job'
|
||||
}, 500);
|
||||
}
|
||||
}
|
||||
|
||||
async cancelJob(c: Context): Promise<Response> {
|
||||
try {
|
||||
const jobId = c.req.param('id');
|
||||
const job = this.orchestrator.getJob(jobId);
|
||||
|
||||
if (!job) {
|
||||
return c.json({
|
||||
success: false,
|
||||
error: 'Job not found'
|
||||
}, 404);
|
||||
}
|
||||
|
||||
if (job.status !== JobStatus.RUNNING && job.status !== JobStatus.PENDING) {
|
||||
return c.json({
|
||||
success: false,
|
||||
error: 'Job cannot be cancelled in current status'
|
||||
}, 400);
|
||||
}
|
||||
|
||||
// Update job status to cancelled
|
||||
job.status = JobStatus.CANCELLED;
|
||||
job.completedAt = new Date();
|
||||
job.error = 'Job cancelled by user';
|
||||
|
||||
logger.info(`Cancelled job: ${jobId}`);
|
||||
|
||||
return c.json({
|
||||
success: true,
|
||||
message: 'Job cancelled successfully',
|
||||
data: job
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to cancel job:', error);
|
||||
|
||||
return c.json({
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Failed to cancel job'
|
||||
}, 500);
|
||||
}
|
||||
}
|
||||
|
||||
async retryJob(c: Context): Promise<Response> {
|
||||
try {
|
||||
const jobId = c.req.param('id');
|
||||
const job = this.orchestrator.getJob(jobId);
|
||||
|
||||
if (!job) {
|
||||
return c.json({
|
||||
success: false,
|
||||
error: 'Job not found'
|
||||
}, 404);
|
||||
}
|
||||
|
||||
if (job.status !== JobStatus.FAILED) {
|
||||
return c.json({
|
||||
success: false,
|
||||
error: 'Only failed jobs can be retried'
|
||||
}, 400);
|
||||
}
|
||||
|
||||
// Create a new job with the same parameters
|
||||
const newJob = await this.orchestrator.runPipeline(job.pipelineId, job.parameters);
|
||||
|
||||
logger.info(`Retried job: ${jobId} as new job: ${newJob.id}`);
|
||||
|
||||
return c.json({
|
||||
success: true,
|
||||
message: 'Job retried successfully',
|
||||
data: {
|
||||
originalJob: job,
|
||||
newJob: newJob
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to retry job:', error);
|
||||
|
||||
return c.json({
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Failed to retry job'
|
||||
}, 500);
|
||||
}
|
||||
}
|
||||
|
||||
async getJobLogs(c: Context): Promise<Response> {
|
||||
try {
|
||||
const jobId = c.req.param('id');
|
||||
const job = this.orchestrator.getJob(jobId);
|
||||
|
||||
if (!job) {
|
||||
return c.json({
|
||||
success: false,
|
||||
error: 'Job not found'
|
||||
}, 404);
|
||||
}
|
||||
|
||||
// In a real implementation, fetch logs from a log store
|
||||
const logs = [
|
||||
{
|
||||
timestamp: job.createdAt,
|
||||
level: 'info',
|
||||
message: `Job ${jobId} created`
|
||||
},
|
||||
...(job.startedAt ? [{
|
||||
timestamp: job.startedAt,
|
||||
level: 'info',
|
||||
message: `Job ${jobId} started`
|
||||
}] : []),
|
||||
...(job.completedAt ? [{
|
||||
timestamp: job.completedAt,
|
||||
level: job.status === JobStatus.COMPLETED ? 'info' : 'error',
|
||||
message: job.status === JobStatus.COMPLETED ?
|
||||
`Job ${jobId} completed successfully` :
|
||||
`Job ${jobId} failed: ${job.error}`
|
||||
}] : [])
|
||||
];
|
||||
|
||||
return c.json({
|
||||
success: true,
|
||||
data: {
|
||||
jobId,
|
||||
logs,
|
||||
totalLogs: logs.length
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get job logs:', error);
|
||||
|
||||
return c.json({
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Failed to get job logs'
|
||||
}, 500);
|
||||
}
|
||||
}
|
||||
|
||||
async getJobMetrics(c: Context): Promise<Response> {
|
||||
try {
|
||||
const jobId = c.req.param('id');
|
||||
const job = this.orchestrator.getJob(jobId);
|
||||
|
||||
if (!job) {
|
||||
return c.json({
|
||||
success: false,
|
||||
error: 'Job not found'
|
||||
}, 404);
|
||||
}
|
||||
|
||||
const metrics = {
|
||||
...job.metrics,
|
||||
duration: job.completedAt && job.startedAt ?
|
||||
job.completedAt.getTime() - job.startedAt.getTime() : null,
|
||||
successRate: job.metrics.recordsProcessed > 0 ?
|
||||
(job.metrics.recordsSuccessful / job.metrics.recordsProcessed) * 100 : 0,
|
||||
errorRate: job.metrics.recordsProcessed > 0 ?
|
||||
(job.metrics.recordsFailed / job.metrics.recordsProcessed) * 100 : 0,
|
||||
status: job.status,
|
||||
startedAt: job.startedAt,
|
||||
completedAt: job.completedAt
|
||||
};
|
||||
|
||||
return c.json({
|
||||
success: true,
|
||||
data: metrics
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get job metrics:', error);
|
||||
|
||||
return c.json({
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Failed to get job metrics'
|
||||
}, 500);
|
||||
}
|
||||
}
|
||||
|
||||
async getJobStats(c: Context): Promise<Response> {
|
||||
try {
|
||||
const jobs = this.orchestrator.listJobs();
|
||||
|
||||
const stats = {
|
||||
total: jobs.length,
|
||||
byStatus: {
|
||||
pending: jobs.filter(j => j.status === JobStatus.PENDING).length,
|
||||
running: jobs.filter(j => j.status === JobStatus.RUNNING).length,
|
||||
completed: jobs.filter(j => j.status === JobStatus.COMPLETED).length,
|
||||
failed: jobs.filter(j => j.status === JobStatus.FAILED).length,
|
||||
cancelled: jobs.filter(j => j.status === JobStatus.CANCELLED).length,
|
||||
},
|
||||
metrics: {
|
||||
totalRecordsProcessed: jobs.reduce((sum, j) => sum + j.metrics.recordsProcessed, 0),
|
||||
totalRecordsSuccessful: jobs.reduce((sum, j) => sum + j.metrics.recordsSuccessful, 0),
|
||||
totalRecordsFailed: jobs.reduce((sum, j) => sum + j.metrics.recordsFailed, 0),
|
||||
averageProcessingTime: jobs.length > 0 ?
|
||||
jobs.reduce((sum, j) => sum + j.metrics.processingTimeMs, 0) / jobs.length : 0,
|
||||
successRate: jobs.length > 0 ?
|
||||
(jobs.filter(j => j.status === JobStatus.COMPLETED).length / jobs.length) * 100 : 0
|
||||
},
|
||||
recentJobs: jobs
|
||||
.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime())
|
||||
.slice(0, 10)
|
||||
.map(job => ({
|
||||
id: job.id,
|
||||
pipelineId: job.pipelineId,
|
||||
status: job.status,
|
||||
createdAt: job.createdAt,
|
||||
processingTime: job.metrics.processingTimeMs,
|
||||
recordsProcessed: job.metrics.recordsProcessed
|
||||
}))
|
||||
};
|
||||
|
||||
return c.json({
|
||||
success: true,
|
||||
data: stats
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get job stats:', error);
|
||||
|
||||
return c.json({
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Failed to get job stats'
|
||||
}, 500);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue