feat(server): 支持稀疏检出路径并完善部署执行队列
- 在部署DTO中添加sparseCheckoutPaths字段支持稀疏检出路径 - 数据模型Deployment新增稀疏检出路径字段及相关数据库映射 - 部署创建时支持设置稀疏检出路径字段 - 部署重试接口实现，支持复制原始部署记录并加入执行队列 - 新增流水线模板初始化与基于模板创建流水线接口 - 优化应用初始化流程，确保执行队列和流水线模板正确加载 - 添加启动日志，提示执行队列初始化完成
This commit is contained in:
233
apps/server/libs/execution-queue.ts
Normal file
233
apps/server/libs/execution-queue.ts
Normal file
@@ -0,0 +1,233 @@
|
||||
import { PipelineRunner } from '../runners/index.ts';
|
||||
import { prisma } from './prisma.ts';
|
||||
|
||||
// 存储正在运行的部署任务
|
||||
const runningDeployments = new Set<number>();
|
||||
|
||||
// 存储待执行的任务队列
|
||||
const pendingQueue: Array<{
|
||||
deploymentId: number;
|
||||
pipelineId: number;
|
||||
}> = [];
|
||||
|
||||
// 定时器ID
|
||||
let pollingTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
// 轮询间隔（毫秒）
|
||||
const POLLING_INTERVAL = 30000; // 30秒
|
||||
|
||||
/**
|
||||
* 执行队列管理器
|
||||
*/
|
||||
export class ExecutionQueue {
|
||||
private static instance: ExecutionQueue;
|
||||
private isProcessing = false;
|
||||
private isPolling = false;
|
||||
|
||||
private constructor() {}
|
||||
|
||||
/**
|
||||
* 获取执行队列的单例实例
|
||||
*/
|
||||
public static getInstance(): ExecutionQueue {
|
||||
if (!ExecutionQueue.instance) {
|
||||
ExecutionQueue.instance = new ExecutionQueue();
|
||||
}
|
||||
return ExecutionQueue.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化执行队列，包括恢复未完成的任务
|
||||
*/
|
||||
public async initialize(): Promise<void> {
|
||||
console.log('Initializing execution queue...');
|
||||
// 恢复未完成的任务
|
||||
await this.recoverPendingDeployments();
|
||||
|
||||
// 启动定时轮询
|
||||
this.startPolling();
|
||||
|
||||
console.log('Execution queue initialized');
|
||||
}
|
||||
|
||||
/**
|
||||
* 从数据库中恢复未完成的部署任务
|
||||
*/
|
||||
private async recoverPendingDeployments(): Promise<void> {
|
||||
try {
|
||||
console.log('Recovering pending deployments from database...');
|
||||
|
||||
// 查询数据库中状态为pending的部署任务
|
||||
const pendingDeployments = await prisma.deployment.findMany({
|
||||
where: {
|
||||
status: 'pending',
|
||||
valid: 1
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
pipelineId: true
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`Found ${pendingDeployments.length} pending deployments`);
|
||||
|
||||
// 将这些任务添加到执行队列中
|
||||
for (const deployment of pendingDeployments) {
|
||||
await this.addTask(deployment.id, deployment.pipelineId);
|
||||
}
|
||||
|
||||
console.log('Pending deployments recovery completed');
|
||||
} catch (error) {
|
||||
console.error('Failed to recover pending deployments:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动定时轮询机制
|
||||
*/
|
||||
private startPolling(): void {
|
||||
if (this.isPolling) {
|
||||
console.log('Polling is already running');
|
||||
return;
|
||||
}
|
||||
|
||||
this.isPolling = true;
|
||||
console.log(`Starting polling with interval ${POLLING_INTERVAL}ms`);
|
||||
|
||||
// 立即执行一次检查
|
||||
this.checkPendingDeployments();
|
||||
|
||||
// 设置定时器定期检查
|
||||
pollingTimer = setInterval(() => {
|
||||
this.checkPendingDeployments();
|
||||
}, POLLING_INTERVAL);
|
||||
}
|
||||
|
||||
/**
|
||||
* 停止定时轮询机制
|
||||
*/
|
||||
public stopPolling(): void {
|
||||
if (pollingTimer) {
|
||||
clearInterval(pollingTimer);
|
||||
pollingTimer = null;
|
||||
this.isPolling = false;
|
||||
console.log('Polling stopped');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查数据库中的待处理部署任务
|
||||
*/
|
||||
private async checkPendingDeployments(): Promise<void> {
|
||||
try {
|
||||
console.log('Checking for pending deployments in database...');
|
||||
|
||||
// 查询数据库中状态为pending的部署任务
|
||||
const pendingDeployments = await prisma.deployment.findMany({
|
||||
where: {
|
||||
status: 'pending',
|
||||
valid: 1
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
pipelineId: true
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`Found ${pendingDeployments.length} pending deployments in polling`);
|
||||
|
||||
// 检查这些任务是否已经在队列中，如果没有则添加
|
||||
for (const deployment of pendingDeployments) {
|
||||
// 检查是否已经在运行队列中
|
||||
if (!runningDeployments.has(deployment.id)) {
|
||||
console.log(`Adding deployment ${deployment.id} to queue from polling`);
|
||||
await this.addTask(deployment.id, deployment.pipelineId);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to check pending deployments:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将部署任务添加到执行队列
|
||||
* @param deploymentId 部署ID
|
||||
* @param pipelineId 流水线ID
|
||||
*/
|
||||
public async addTask(deploymentId: number, pipelineId: number): Promise<void> {
|
||||
// 检查是否已经在运行队列中
|
||||
if (runningDeployments.has(deploymentId)) {
|
||||
console.log(`Deployment ${deploymentId} is already queued or running`);
|
||||
return;
|
||||
}
|
||||
|
||||
// 添加到运行队列
|
||||
runningDeployments.add(deploymentId);
|
||||
|
||||
// 添加到待执行队列
|
||||
pendingQueue.push({ deploymentId, pipelineId });
|
||||
|
||||
// 开始处理队列（如果尚未开始）
|
||||
if (!this.isProcessing) {
|
||||
this.processQueue();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理执行队列中的任务
|
||||
*/
|
||||
private async processQueue(): Promise<void> {
|
||||
this.isProcessing = true;
|
||||
|
||||
while (pendingQueue.length > 0) {
|
||||
const task = pendingQueue.shift();
|
||||
|
||||
if (task) {
|
||||
try {
|
||||
// 执行流水线
|
||||
await this.executePipeline(task.deploymentId, task.pipelineId);
|
||||
} catch (error) {
|
||||
console.error('执行流水线失败:', error);
|
||||
// 这里可以添加更多的错误处理逻辑
|
||||
} finally {
|
||||
// 从运行队列中移除
|
||||
runningDeployments.delete(task.deploymentId);
|
||||
}
|
||||
}
|
||||
|
||||
// 添加一个小延迟以避免过度占用资源
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
|
||||
this.isProcessing = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行流水线
|
||||
* @param deploymentId 部署ID
|
||||
* @param pipelineId 流水线ID
|
||||
*/
|
||||
private async executePipeline(deploymentId: number, pipelineId: number): Promise<void> {
|
||||
try {
|
||||
const runner = new PipelineRunner(deploymentId);
|
||||
await runner.run(pipelineId);
|
||||
} catch (error) {
|
||||
console.error('执行流水线失败:', error);
|
||||
// 错误处理可以在这里添加，比如更新部署状态为失败
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取队列状态
|
||||
*/
|
||||
public getQueueStatus(): {
|
||||
pendingCount: number;
|
||||
runningCount: number;
|
||||
} {
|
||||
return {
|
||||
pendingCount: pendingQueue.length,
|
||||
runningCount: runningDeployments.size
|
||||
};
|
||||
}
|
||||
}
|
||||
247
apps/server/libs/pipeline-template.ts
Normal file
247
apps/server/libs/pipeline-template.ts
Normal file
@@ -0,0 +1,247 @@
|
||||
import { prisma } from './prisma.ts';
|
||||
|
||||
// 默认流水线模板
|
||||
export interface PipelineTemplate {
|
||||
name: string;
|
||||
description: string;
|
||||
steps: Array<{
|
||||
name: string;
|
||||
order: number;
|
||||
script: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
// 系统默认的流水线模板
|
||||
export const DEFAULT_PIPELINE_TEMPLATES: PipelineTemplate[] = [
|
||||
{
|
||||
name: 'Git Clone Pipeline',
|
||||
description: '默认的Git克隆流水线，用于从仓库克隆代码',
|
||||
steps: [
|
||||
{
|
||||
name: 'Clone Repository',
|
||||
order: 0,
|
||||
script: '# 克隆指定commit的代码\ngit init\ngit remote add origin $REPOSITORY_URL\ngit fetch --depth 1 origin $COMMIT_HASH\ngit checkout -q FETCH_HEAD\n\n# 显示当前提交信息\ngit log --oneline -1',
|
||||
},
|
||||
{
|
||||
name: 'Install Dependencies',
|
||||
order: 1,
|
||||
script: '# 安装项目依赖\nnpm install',
|
||||
},
|
||||
{
|
||||
name: 'Run Tests',
|
||||
order: 2,
|
||||
script: '# 运行测试\nnpm test',
|
||||
},
|
||||
{
|
||||
name: 'Build Project',
|
||||
order: 3,
|
||||
script: '# 构建项目\nnpm run build',
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
name: 'Sparse Checkout Pipeline',
|
||||
description: '稀疏检出流水线，适用于monorepo项目，只获取指定目录的代码',
|
||||
steps: [
|
||||
{
|
||||
name: 'Sparse Checkout Repository',
|
||||
order: 0,
|
||||
script: '# 进行稀疏检出指定目录的代码\ngit init\ngit remote add origin $REPOSITORY_URL\ngit config core.sparseCheckout true\necho "$SPARSE_CHECKOUT_PATHS" > .git/info/sparse-checkout\ngit fetch --depth 1 origin $COMMIT_HASH\ngit checkout -q FETCH_HEAD\n\n# 显示当前提交信息\ngit log --oneline -1',
|
||||
},
|
||||
{
|
||||
name: 'Install Dependencies',
|
||||
order: 1,
|
||||
script: '# 安装项目依赖\nnpm install',
|
||||
},
|
||||
{
|
||||
name: 'Run Tests',
|
||||
order: 2,
|
||||
script: '# 运行测试\nnpm test',
|
||||
},
|
||||
{
|
||||
name: 'Build Project',
|
||||
order: 3,
|
||||
script: '# 构建项目\nnpm run build',
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
name: 'Simple Deploy Pipeline',
|
||||
description: '简单的部署流水线，包含基本的构建和部署步骤',
|
||||
steps: [
|
||||
{
|
||||
name: 'Clone Repository',
|
||||
order: 0,
|
||||
script: '# 克隆指定commit的代码\ngit init\ngit remote add origin $REPOSITORY_URL\ngit fetch --depth 1 origin $COMMIT_HASH\ngit checkout -q FETCH_HEAD',
|
||||
},
|
||||
{
|
||||
name: 'Build and Deploy',
|
||||
order: 1,
|
||||
script: '# 构建并部署项目\nnpm run build\n\n# 部署到目标服务器\n# 这里可以添加具体的部署命令',
|
||||
}
|
||||
]
|
||||
}
|
||||
];
|
||||
|
||||
/**
|
||||
* 初始化系统默认流水线模板
|
||||
*/
|
||||
export async function initializePipelineTemplates(): Promise<void> {
|
||||
console.log('Initializing pipeline templates...');
|
||||
|
||||
try {
|
||||
// 检查是否已经存在模板流水线
|
||||
const existingTemplates = await prisma.pipeline.findMany({
|
||||
where: {
|
||||
name: {
|
||||
in: DEFAULT_PIPELINE_TEMPLATES.map(template => template.name)
|
||||
},
|
||||
valid: 1
|
||||
}
|
||||
});
|
||||
|
||||
// 如果没有现有的模板，则创建默认模板
|
||||
if (existingTemplates.length === 0) {
|
||||
console.log('Creating default pipeline templates...');
|
||||
|
||||
for (const template of DEFAULT_PIPELINE_TEMPLATES) {
|
||||
// 创建模板流水线（使用负数ID表示模板）
|
||||
const pipeline = await prisma.pipeline.create({
|
||||
data: {
|
||||
name: template.name,
|
||||
description: template.description,
|
||||
createdBy: 'system',
|
||||
updatedBy: 'system',
|
||||
valid: 1,
|
||||
projectId: null // 模板不属于任何特定项目
|
||||
}
|
||||
});
|
||||
|
||||
// 创建模板步骤
|
||||
for (const step of template.steps) {
|
||||
await prisma.step.create({
|
||||
data: {
|
||||
name: step.name,
|
||||
order: step.order,
|
||||
script: step.script,
|
||||
pipelineId: pipeline.id,
|
||||
createdBy: 'system',
|
||||
updatedBy: 'system',
|
||||
valid: 1
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`Created template: ${template.name}`);
|
||||
}
|
||||
} else {
|
||||
console.log('Pipeline templates already exist, skipping initialization');
|
||||
}
|
||||
|
||||
console.log('Pipeline templates initialization completed');
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize pipeline templates:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有可用的流水线模板
|
||||
*/
|
||||
export async function getAvailableTemplates(): Promise<Array<{id: number, name: string, description: string}>> {
|
||||
try {
|
||||
const templates = await prisma.pipeline.findMany({
|
||||
where: {
|
||||
projectId: null, // 模板流水线没有关联的项目
|
||||
valid: 1
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
description: true
|
||||
}
|
||||
});
|
||||
|
||||
// 处理可能为null的description字段
|
||||
return templates.map(template => ({
|
||||
id: template.id,
|
||||
name: template.name,
|
||||
description: template.description || ''
|
||||
}));
|
||||
} catch (error) {
|
||||
console.error('Failed to get pipeline templates:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 基于模板创建新的流水线
|
||||
* @param templateId 模板ID
|
||||
* @param projectId 项目ID
|
||||
* @param pipelineName 新流水线名称
|
||||
* @param pipelineDescription 新流水线描述
|
||||
*/
|
||||
export async function createPipelineFromTemplate(
|
||||
templateId: number,
|
||||
projectId: number,
|
||||
pipelineName: string,
|
||||
pipelineDescription: string
|
||||
): Promise<number> {
|
||||
try {
|
||||
// 获取模板流水线及其步骤
|
||||
const templatePipeline = await prisma.pipeline.findUnique({
|
||||
where: {
|
||||
id: templateId,
|
||||
projectId: null, // 确保是模板流水线
|
||||
valid: 1
|
||||
},
|
||||
include: {
|
||||
steps: {
|
||||
where: {
|
||||
valid: 1
|
||||
},
|
||||
orderBy: {
|
||||
order: 'asc'
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (!templatePipeline) {
|
||||
throw new Error(`Template with id ${templateId} not found`);
|
||||
}
|
||||
|
||||
// 创建新的流水线
|
||||
const newPipeline = await prisma.pipeline.create({
|
||||
data: {
|
||||
name: pipelineName,
|
||||
description: pipelineDescription,
|
||||
projectId: projectId,
|
||||
createdBy: 'system',
|
||||
updatedBy: 'system',
|
||||
valid: 1
|
||||
}
|
||||
});
|
||||
|
||||
// 复制模板步骤到新流水线
|
||||
for (const templateStep of templatePipeline.steps) {
|
||||
await prisma.step.create({
|
||||
data: {
|
||||
name: templateStep.name,
|
||||
order: templateStep.order,
|
||||
script: templateStep.script,
|
||||
pipelineId: newPipeline.id,
|
||||
createdBy: 'system',
|
||||
updatedBy: 'system',
|
||||
valid: 1
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`Created pipeline from template ${templateId}: ${newPipeline.name}`);
|
||||
return newPipeline.id;
|
||||
} catch (error) {
|
||||
console.error('Failed to create pipeline from template:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user