EPESS/epess-web-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: update dependencies and enhance document schema

Browse Source 
- Updated package.json and package-lock.json to include the new dependency 'quill-to-pdf' for improved document export functionality.
- Modified DocumentSchema to introduce a new 'DocumentExportObject' type, facilitating document export operations.
- Cleaned up commented-out code in document.service.ts and minio.service.ts for better readability and maintainability.
- Adjusted quiz schema to expose user input and questions as JSON types, enhancing data flexibility.
- Updated workshop subscription logic to maintain accurate participant counts upon new subscriptions.

These changes improve the overall functionality and maintainability of the project, ensuring better document handling and schema consistency.
This commit is contained in:
Ly Tuan Kiet
2024-12-11 16:31:16 +07:00
parent 1fcc7b9a5f
commit 871f24edb0
8 changed files with 201 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
package-lock.json generated
View File

@@ -67,6 +67,7 @@
"openai": "^4.76.0",
"passport-jwt": "^4.0.1",
"quill": "^2.0.3",
"quill-to-pdf": "^1.0.7",
"reflect-metadata": "^0.2.2",
"rxjs": "^7.8.1",
"swagger-ui-express": "^5.0.1",
@@ -15230,6 +15231,21 @@
"node": ">= 12.0.0"
}
},
"node_modules/quill-to-pdf": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/quill-to-pdf/-/quill-to-pdf-1.0.7.tgz",
"integrity": "sha512-aHB4m7r9Vb0da2JldFSMGge2Sa8oUYFBMxXl0Ne/p9O33Xav+jkqKsE6yYLfw+3BLtKwqJyEUv9bzYvmvU746w==",
"license": "MIT",
"dependencies": {
"quilljs-parser": "^1.0.11"
}
},
"node_modules/quilljs-parser": {
"version": "1.0.14",
"resolved": "https://registry.npmjs.org/quilljs-parser/-/quilljs-parser-1.0.14.tgz",
"integrity": "sha512-tycXmRQRcAp3Mq4vb+8Je+u8s3zoBEtwJmc3dZn4N6rFDYdBl1LlkzXJMBCvnpapHyejoOIViYq558MF2Ca7hg==",
"license": "MIT"
},
"node_modules/randombytes": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",

11
package.json
View File

@@ -89,6 +89,7 @@
"openai": "^4.76.0",
"passport-jwt": "^4.0.1",
"quill": "^2.0.3",
"quill-to-pdf": "^1.0.7",
"reflect-metadata": "^0.2.2",
"rxjs": "^7.8.1",
"swagger-ui-express": "^5.0.1",
@@ -139,13 +140,19 @@
"@css-inline/css-inline-linux-x64-musl": "^0.14.3"
},
"jest": {
"moduleFileExtensions": ["js", "json", "ts"],
"moduleFileExtensions": [
"js",
"json",
"ts"
],
"rootDir": "src",
"testRegex": ".*\\.spec\\.ts$",
"transform": {
"^.+\\.(t|j)s$": "ts-jest"
},
"collectCoverageFrom": ["**/*.(t|j)s"],
"collectCoverageFrom": [
"**/*.(t|j)s"
],
"coverageDirectory": "../coverage",
"testEnvironment": "node"
},

58
src/Document/document.schema.ts
View File

@@ -100,6 +100,23 @@ export class DocumentSchema extends PothosSchema {
})
}
@PothosRef()
DocumentExportObject() {
return this.builder.simpleObject('DocumentExportObject', {
fields: (t) => ({
documentId: t.string(),
pageIndex: t.int(),
type: t.field({
type: this.builder.enumType('DocumentExportType', {
values: ['PDF', 'DOCX'] as const,
}),
nullable: false,
}),
fileUrl: t.string(),
}),
})
}
@PothosRef()
documentDeltaInput() {
return this.builder.inputType('DocumentDeltaInput', {
@@ -179,6 +196,45 @@ export class DocumentSchema extends PothosSchema {
return document
},
}),
// exportDocument: t.field({
// type: this.DocumentExportObject(),
// args: {
// documentId: t.arg({ type: 'String', required: true }),
// },
// resolve: async (_query, args, ctx: SchemaContext) => {
// if (ctx.isSubscription) {
// throw new Error('Not allowed')
// }
// if (!args.documentId) {
// throw new Error('Document id not found')
// }
// const document = await this.prisma.document.findUnique({
// where: { id: args.documentId },
// include: {
// collaborators: true,
// },
// })
// if (!document) {
// throw new Error('Document not found')
// }
// // check if user is owner or collaborator
// if (
// document.ownerId !== ctx.http?.me?.id &&
// !document.collaborators.some((c) => c.userId === ctx.http?.me?.id && c.readable)
// ) {
// throw new Error('User is not owner or collaborator of document')
// }
// // export document
// const fileUrl = await this.minio.exportDocument(args.documentId)
// return {
// documentId: args.documentId,
// pageIndex: 0,
// type: args.type,
// fileUrl: '',
// }
// },
// }),
eventDocumentClientRequestSync: t.field({
type: this.documentDelta(),
args: {
@@ -547,7 +603,7 @@ export class DocumentSchema extends PothosSchema {
// 0.5% chance to request sync
if (random <= 0.005) {
// check grammar too
this.documentService.checkGrammarForPage(payload.documentId, payload.pageIndex)
// this.documentService.checkGrammarForPage(payload.documentId, payload.pageIndex)
Logger.log('request sync', 'request sync')
payload.requestSync = true
return payload

85
src/Document/document.service.ts
View File

@@ -98,50 +98,51 @@ export class DocumentService {
}
// check grammar for a page by parallely send each sentence to OpenAI service as text and get the result, after that return the result as delta and publish the result to the document
async checkGrammarForPage(documentId: string, pageId: number): Promise<void> {
const sentenceDelta = await this.pageToSentence(documentId, pageId)
// async checkGrammarForPage(documentId: string, pageId: number): Promise<void> {
// const sentenceDelta = await this.pageToSentence(documentId, pageId)
// Extract the entire page content as a single text
const pageText = sentenceDelta.ops
.filter((op) => typeof op.insert === 'string')
.map((op) => op.insert as string)
.join(' ')
// // Extract the entire page content as a single text
// const pageText = sentenceDelta.ops
// .filter((op) => typeof op.insert === 'string')
// .map((op) => op.insert as string)
// .join(' ')
// Create a unique cache key for the entire page
const cacheKey = `grammar_check:${documentId}:${pageId}`
// // Create a unique cache key for the entire page
// const cacheKey = `grammar_check:${documentId}:${pageId}`
// Try to get cached result first
const cachedResult = await this.redis.get(cacheKey)
// // Try to get cached result first
// const cachedResult = await this.redis.get(cacheKey)
if (cachedResult) {
Logger.log('Cached grammar check result exists', 'Grammar Check')
return
}
// Process the entire page text
const grammarCheckResult = await this.openai.processText(pageText, 0, PromptType.CHECK_GRAMMAR)
if (!grammarCheckResult.result) {
return
}
// Cache the result
await this.redis.setPermanent(cacheKey, JSON.stringify(grammarCheckResult))
// Calculate diff between original page and corrected page
const diff = await this.diff(pageText, grammarCheckResult.result)
// Build payload
// Publish the result to the subscriber
this.pubSub.publish(`${DocumentEvent.AI_SUGGESTION}.${documentId}.${pageId}`, payload)
}
async diff(original: string, corrected: string): Promise<Delta> {
const originalDelta = new Delta().insert(original)
const correctedDelta = new Delta().insert(corrected)
const diff = originalDelta.diff(correctedDelta)
Logger.log(diff, 'diff')
return diff
}
// if (cachedResult) {
// Logger.log('Cached grammar check result exists', 'Grammar Check')
// return
// }
// // Process the entire page text
// const grammarCheckResult = await this.openai.processText(pageText, 0, PromptType.CHECK_GRAMMAR)
// if (!grammarCheckResult.result) {
// return
// }
// // Cache the result
// await this.redis.setPermanent(cacheKey, JSON.stringify(grammarCheckResult))
// // Calculate diff between original page and corrected page
// const diff = await this.diff(pageText, grammarCheckResult.result)
// // Build payload
// // Publish the result to the subscriber
// this.pubSub.publish(`${DocumentEvent.AI_SUGGESTION}.${documentId}.${pageId}`, payload)
// }
// async diff(original: string, corrected: string): Promise<Delta> {
// const originalDelta = new Delta().insert(original)
// const correctedDelta = new Delta().insert(corrected)
// const diff = originalDelta.diff(correctedDelta)
// Logger.log(diff, 'diff')
// return diff
// }
}

73
src/Minio/minio.service.ts
View File

@@ -5,6 +5,7 @@ import { FileUpload } from 'graphql-upload/processRequest.mjs'
import { BucketItem, Client } from 'minio'
import { MINIO_CONNECTION } from 'nestjs-minio'
import Delta from 'quill-delta'
// import { pdfExporter } from 'quill-to-pdf'
import { DateTimeUtils } from 'src/common/utils/datetime.utils'
import { v4 as uuidv4 } from 'uuid'
@Injectable()
@@ -33,7 +34,7 @@ export class MinioService {
}
async upsertFile(name: string, file: FileUpload, category: string) {
const { mimetype, createReadStream, filename: actualFileName } = await file
const { mimetype, createReadStream, filename: _actualFileName } = await file
const fileBuffer = createReadStream()
@@ -150,15 +151,67 @@ export class MinioService {
return JSON.parse(buffer.toString()) as Delta
}
// export document to docx format by get all pages and convert to docx
async exportDocument(id: string) {
// get all pages
const _pages = await this.minioClient.listObjects(
this.configService.get('BUCKET_NAME') ?? 'epess',
`documents/${id}`,
)
// convert to docx
// const docx = await this.convertToDocx(pages)
}
// async exportDocument(id: string) {
// try {
// // List all page objects for the document
// const stream = this.minioClient.listObjects(
// this.configService.get('BUCKET_NAME') ?? 'epess',
// `documents/${id}/`,
// true,
// )
// // Collect page items
// const pageItems: BucketItem[] = await new Promise((resolve, reject) => {
// const items: BucketItem[] = []
// stream.on('data', (item) => items.push(item))
// stream.on('end', () => resolve(items))
// stream.on('error', (err) => reject(err))
// })
// // Sort pages numerically (assuming page names are numeric)
// const sortedPageItems = pageItems
// .filter((item) => item.name && /\/\d+$/.test(item.name))
// .sort((a, b) => {
// const pageA = parseInt(a.name?.split('/').pop() ?? '0')
// const pageB = parseInt(b.name?.split('/').pop() ?? '0')
// return pageA - pageB
// })
// // Fetch and parse page deltas in order
// const pages = await Promise.all(
// sortedPageItems.map(async (page) => {
// const delta = await this.minioClient.getObject(
// this.configService.get('BUCKET_NAME') ?? 'epess',
// page.name ?? '',
// )
// const buffer = await delta.read()
// return JSON.parse(buffer.toString()) as Delta
// }),
// )
// // Convert to PDF
// const pdf = await pdfExporter.generatePdf(pages)
// // convert blob to
// const pdfBuffer = await pdf.arrayBuffer().then((buffer) => Buffer.from(buffer))
// // Upload PDF to Minio
// await this.minioClient.putObject(
// this.configService.get('BUCKET_NAME') ?? 'epess',
// `documents/${id}/export.pdf`,
// pdfBuffer,
// undefined,
// {
// 'Content-Type': 'application/pdf',
// },
// )
// // Get and return document URL
// return await this.getFileUrl(`documents/${id}/export.pdf`, 'documents')
// } catch (error) {
// Logger.error(`Failed to export document ${id}: ${error}`, 'MinioService')
// throw error
// }
// }
fileName() {
// generate a unique file name using uuid
return uuidv4()

8
src/Quiz/quiz.schema.ts
View File

@@ -85,8 +85,12 @@ export class QuizSchema extends PothosSchema {
numberOfQuestions: t.exposeInt('numberOfQuestions'),
correctPoints: t.exposeInt('correctPoints'),
totalPoints: t.exposeInt('totalPoints'),
userInput: t.exposeStringList('userInput'),
questions: t.exposeStringList('questions'),
userInput: t.expose('userInput', {
type: 'Json',
}),
questions: t.expose('questions', {
type: 'Json',
}),
createdAt: t.expose('createdAt', {
type: 'DateTime',
}),

15
src/WorkshopSubscription/workshopsubscription.schema.ts
View File

@@ -1,4 +1,4 @@
import { Inject, Injectable } from '@nestjs/common'
import { Inject, Injectable, Logger } from '@nestjs/common'
import { Pothos, PothosRef, PothosSchema, SchemaBuilderToken } from '@smatch-corp/nestjs-pothos'
import { DateTimeUtils } from 'src/common/utils/datetime.utils'
import { Builder } from '../Graphql/graphql.builder'
@@ -126,12 +126,23 @@ export class WorkshopSubscriptionSchema extends PothosSchema {
throw new Error('User already subscribed to workshop')
}
// create the workshop subscription
return await this.prisma.workshopSubscription.create({
const result = await this.prisma.workshopSubscription.create({
data: {
userId,
workshopId: args.workshopId,
},
})
// update participant count by querying the workshop subscription
const participantCount = await this.prisma.workshopSubscription.count({
where: { workshopId: args.workshopId },
})
await this.prisma.workshop.update({
where: { id: args.workshopId },
data: {
registeredParticipants: participantCount,
},
})
return result
},
}),
}))

2
src/types/pothos.generated.ts
View File

File diff suppressed because one or more lines are too long