EPESS/epess-web-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement AI suggestion

Browse Source
This commit is contained in:
Ly Tuan Kiet
2024-12-12 21:22:50 +07:00
parent 3652fda7d3
commit 68353d8985
7 changed files with 133 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/CenterMentor/centermentor.schema.ts
View File

@@ -193,28 +193,6 @@ export class CenterMentorSchema extends PothosSchema {
})
},
}),
testInviteCenterMentor: t.prismaField({
type: this.centerMentor(),
args: {
email: t.arg({ type: 'String', required: true }),
centerId: t.arg({ type: 'String', required: true }),
},
description: 'Test invite center mentor.',
resolve: async (_query, _root, args) => {
return this.prisma.$transaction(async () => {
// sign token
const token = this.jwtUtils.signTokenRS256({ centerId: args.centerId, email: args.email }, '1d')
// build invite url
const inviteUrl = `${process.env.CENTER_BASE_URL}/invite?token=${token}`
// mail to user with params centerId, email
await this.mailService.sendTemplateEmail([args.email], 'Invite to center', 'MentorInvitation', {
center_name: args.centerId,
invite_url: inviteUrl,
})
return null
})
},
}),
approveOrRejectCenterMentor: t.prismaField({
type: this.centerMentor(),
description: 'Approve or reject a center mentor.',

17
src/Cron/cron.service.ts
View File

@@ -193,9 +193,12 @@ export class CronService {
@Cron(CronExpression.EVERY_DAY_AT_1AM)
async taskDisableServiceWithoutSchedule() {
Logger.log('Disabling service without any schedule', 'CronService')
const services = await this.prisma.managedService.findMany({
const services = await this.prisma.service.findMany({
where: {
NOT: {
// check if service has any schedule in the past 30 days
managedService: {
some: {
schedule: {
some: {
scheduleStart: { gte: DateTimeUtils.now().minus({ days: 30 }).toJSDate() },
@@ -203,18 +206,20 @@ export class CronService {
},
},
},
// and createdAt is more than 3 days ago
createdAt: {
lt: DateTimeUtils.now().minus({ days: 3 }).toJSDate(),
},
},
},
})
for (const service of services) {
await this.prisma.managedService.update({
await this.prisma.service.update({
where: { id: service.id },
data: {
service: {
update: {
status: ServiceStatus.INACTIVE,
},
},
},
})
Logger.log(`Service ${service.id} has been disabled`, 'CronService')
}

200
src/Document/document.service.ts
View File

@@ -3,145 +3,119 @@ import { Inject, Injectable, forwardRef } from '@nestjs/common'
import { Logger } from '@nestjs/common'
import Delta, { Op } from 'quill-delta'
import { OpenaiService, PromptType } from 'src/OpenAI/openai.service'
import { PrismaService } from 'src/Prisma/prisma.service'
import { PubSubService } from 'src/PubSub/pubsub.service'
import { RedisService } from 'src/Redis/redis.service'
import { MinioService } from '../Minio/minio.service'
import { DocumentEvent } from './document.event'
import { DocumentDelta } from './document.type'
export type GrammarCheckResult = {
original: string
corrected: string
pageId: number
documentId: string
delta: Delta
}
enum customAttributes {
AI_SUGGESTION = 'ai-suggestion',
}
export const blacklist = [
'!',
'?',
'.',
',',
':',
';',
'(',
')',
'[',
']',
'{',
'}',
'|',
'\\',
'/',
"'",
'"',
'`',
'~',
'^',
'&',
'*',
'=',
'+',
'-',
'_',
' ',
'\n',
'\n\n',
'\t',
'\r',
'\v',
'\f',
]
@Injectable()
export class DocumentService {
constructor(
private readonly prisma: PrismaService,
private readonly minio: MinioService,
private readonly openai: OpenaiService,
private readonly redis: RedisService,
private readonly pubSub: PubSubService,
) {}
// page to list paragraph as delta
async pageToParagraph(content: Delta): Promise<Delta> {
// Create a new Delta to store paragraphs
const paragraphDelta = new Delta()
// Iterate through the operations in the content Delta
for (const op of content.ops) {
// If the operation is a string insert
if (typeof op.insert === 'string') {
// Split the text by newline characters
const paragraphs = op.insert.split('\n').filter((p) => p.trim() !== '')
// Add each non-empty paragraph as a separate insert with a newline
paragraphs.forEach((paragraph, index) => {
paragraphDelta.insert(paragraph)
// Add a newline after each paragraph except the last one
if (index < paragraphs.length - 1) {
paragraphDelta.insert('\n')
}
})
} else {
// If the operation is not a string (e.g., an embedded object),
// add it to the paragraphDelta as-is
paragraphDelta.push(op)
}
}
return paragraphDelta
}
async paragraphToSentence(content: Delta): Promise<Delta> {
// Create a new Delta to store sentences
const sentenceDelta = new Delta()
// Iterate through the operations in the content Delta
for (const op of content.ops) {
// If the operation is a string insert
if (typeof op.insert === 'string') {
// Split the text into paragraphs first
const paragraphs = op.insert.split('\n').filter((p) => p.trim() !== '')
paragraphs.forEach((paragraph, paragraphIndex) => {
// Split paragraph into sentences using regex
// This handles common sentence-ending punctuation: . ! ?
const sentences = paragraph.split(/(?<=[.!?])\s+/).filter((s) => s.trim() !== '')
sentences.forEach((sentence, sentenceIndex) => {
sentenceDelta.insert(sentence.trim())
// Add a newline after each sentence except the last one in the paragraph
if (sentenceIndex < sentences.length - 1) {
sentenceDelta.insert('\n')
}
})
// Add a newline between paragraphs, except after the last paragraph
if (paragraphIndex < paragraphs.length - 1) {
sentenceDelta.insert('\n')
}
})
} else {
// If the operation is not a string (e.g., an embedded object),
// add it to the sentenceDelta as-is
sentenceDelta.push(op)
}
}
return sentenceDelta
}
async pageToSentence(documentId: string, pageId: number): Promise<Delta> {
const content = await this.minio.getDocumentContent(documentId, pageId)
const paragraphDelta = await this.pageToParagraph(content)
const sentenceDelta = await this.paragraphToSentence(paragraphDelta)
return sentenceDelta
}
// check grammar for a page by parallely send each sentence to OpenAI service as text and get the result, after that return the result as delta and publish the result to the document
async checkGrammarForPage(documentId: string, pageId: number): Promise<void> {
const sentenceDelta = await this.pageToSentence(documentId, pageId)
// Extract the entire page content as a single text
const pageText = sentenceDelta.ops
.filter((op) => typeof op.insert === 'string')
.map((op) => op.insert as string)
.join(' ')
// Create a unique cache key for the entire page
const cacheKey = `grammar_check:${documentId}:${pageId}`
// Try to get cached result first
const cachedResult = await this.redis.get(cacheKey)
if (cachedResult) {
Logger.log('Cached grammar check result exists', 'Grammar Check')
const content = await this.minio.getDocumentContent(documentId, pageId)
content.ops.forEach(async (op) => {
if (typeof op.insert !== 'string') {
return
}
// Process the entire page text
const grammarCheckResult = await this.openai.processText(pageText, 0, PromptType.CHECK_GRAMMAR)
if (!grammarCheckResult.result) {
if (!this.isSentence(op) && blacklist.includes(op.insert)) {
return
}
// check if the sentence is already corrected by checking the attributes
if (op.attributes?.[customAttributes.AI_SUGGESTION]) {
return
}
const originalDelta = new Delta().push(op)
const grammarCheckResult = await this.openai.processText(op.insert, PromptType.CHECK_GRAMMAR)
if (!grammarCheckResult) {
return
}
// create new delta and maintain the original delta attributes
const newDelta = new Delta().push(op)
newDelta.ops[0].attributes = originalDelta.ops[0].attributes
newDelta.ops[0].insert = grammarCheckResult
// compose the original delta with the grammarCheckResult
const correctedDelta = originalDelta.compose(newDelta)
// calculate where to insert the correctedDelta
const index = content.ops.findIndex((op) => op.insert === op.insert)
content.ops.splice(index, 0, correctedDelta.ops[0])
Logger.log(JSON.stringify(content), 'content')
// Cache the result
await this.redis.setPermanent(cacheKey, JSON.stringify(grammarCheckResult))
// Calculate diff between original page and corrected page
const diff = await this.diff(pageText, grammarCheckResult.result)
// Build payload
// Publish the result to the subscriber
// publish the result to the subscriber
const payload: DocumentDelta = {
documentId,
pageIndex: pageId,
eventType: DocumentEvent.AI_SUGGESTION,
delta: content,
senderId: 'system',
}
this.pubSub.publish(`${DocumentEvent.AI_SUGGESTION}.${documentId}.${pageId}`, payload)
})
}
async diff(original: string, corrected: string): Promise<Delta> {
const originalDelta = new Delta().insert(original)
const correctedDelta = new Delta().insert(corrected)
const diff = originalDelta.diff(correctedDelta)
Logger.log(diff, 'diff')
return diff
isSentence(op: Op) {
if (typeof op.insert !== 'string') {
return false
}
return op.insert?.match(/^[A-Z]/i)
}
}

5
src/Document/document.type.ts
View File

@@ -1,9 +1,12 @@
import Delta from 'quill-delta'
import { DocumentEvent } from './document.event'
export type DocumentDelta = Delta & {
export type DocumentDelta = {
pageIndex: number
documentId: string
senderId?: string
requestSync?: boolean
totalPage?: number
delta: Delta
eventType: DocumentEvent
}

8
src/OpenAI/openai.service.ts
View File

@@ -59,7 +59,7 @@ export class OpenaiService {
return response.choices[0].message.content
}
async processText(text: string, sentenceIndex: number, prompt: PromptType) {
async processText(text: string, prompt: PromptType) {
const systemPrompt = 'You are an expert proofreader, the most advanced AI tool on the planet.'
const userPrompt = prompts[prompt].replace('{text}', text)
try {
@@ -74,11 +74,7 @@ export class OpenaiService {
const result = response.choices[0].message.content
// split to get data only in triple quotes
const resultData = result?.split('"""')[1]
Logger.log(resultData, 'result')
return {
sentenceIndex,
result: resultData,
}
return resultData
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
} catch (error: any) {
Logger.error(`Error in text processing: ${error.message}`, 'OpenaiService')

13
src/Quiz/quiz.schema.ts
View File

@@ -190,6 +190,9 @@ export class QuizSchema extends PothosSchema {
if (!schedule) {
throw new Error('Schedule not found')
}
if (schedule.customerId !== ctx.http.me.id) {
throw new Error('Unauthorized')
}
// get centerMentorId from schedule
const centerMentorId = schedule.managedService.mentorId
if (!centerMentorId) {
@@ -202,6 +205,16 @@ export class QuizSchema extends PothosSchema {
centerMentorId: centerMentorId,
},
})
// check if user has already taken the quiz
const quizAttempt = await this.prisma.quizAttempt.findFirst({
where: {
userId: ctx.http.me.id,
quizId: quizzes[0]?.id,
},
})
if (quizAttempt) {
throw new Error('User has already taken the quiz')
}
// get amount of questions using nrOfQuestions and random index based on random
const randomIndex = quizzes.length > 0 ? Math.floor(random * quizzes.length) : 0
const nrOfQuestions = quizzes[0]?.nrOfQuestions ?? 1

19
src/Service/service.schema.ts
View File

@@ -143,25 +143,6 @@ export class ServiceSchema extends PothosSchema {
@Pothos()
init() {
this.builder.queryFields((t) => ({
testServices: t.prismaConnection(
{
description: 'A test connection for services',
type: this.service(),
cursor: 'id',
args: this.builder.generator.findManyArgs('Service'),
resolve: async (query, _root, _args, _ctx, _info) => {
return await this.prisma.service.findMany({
...query,
})
},
totalCount: (query) => {
return this.prisma.service.count({
...query,
})
},
},
{},
),
services: t.prismaField({
description: 'Retrieve a list of services with optional filtering, ordering, and pagination.',
type: [this.service()],