feat: enhance document processing with AI suggestions and preview image support

- Added AI_SUGGESTION event to DocumentEvent enum for improved document interaction.
- Implemented updateDocumentPreviewImage mutation in DocumentSchema to allow updating of document preview images.
- Integrated grammar checking functionality in DocumentService, utilizing OpenAI for real-time suggestions and caching results in Redis.
- Enhanced MinioService with methods for file upsert and document content retrieval.
- Updated PrismaTypes to include new relations and fields for better data structure alignment.
- Commented out unused RESTful service methods for future cleanup.

These changes improve the document editing experience by leveraging AI capabilities and enhancing the schema for better data management.

src/Document/document.event.ts

@@ -7,4 +7,5 @@ export enum DocumentEvent {
   ACTIVE_DOCUMENT_ID_CHANGED = 'document_active_document_id_changed',
   CLIENT_REQUEST_SYNC = 'document_client_request_sync',
   SERVER_REQUEST_SYNC = 'document_server_request_sync',
+  AI_SUGGESTION = 'document_ai_suggestion',
 }

src/Document/document.schema.ts

@@ -6,6 +6,7 @@ import { MinioService } from 'src/Minio/minio.service'
 import { Builder, SchemaContext } from '../Graphql/graphql.builder'
 import { PrismaService } from '../Prisma/prisma.service'
 import { DocumentEvent } from './document.event'
+import { DocumentService } from './document.service'
 import { DocumentDelta } from './document.type'
 @Injectable()
 export class DocumentSchema extends PothosSchema {
@@ -13,6 +14,7 @@ export class DocumentSchema extends PothosSchema {
     @Inject(SchemaBuilderToken) private readonly builder: Builder,
     private readonly prisma: PrismaService,
     private readonly minio: MinioService,
+    private readonly documentService: DocumentService,
   ) {
     super()
   }
@@ -24,6 +26,7 @@ export class DocumentSchema extends PothosSchema {
         id: t.exposeID('id'),
         name: t.exposeString('name'),
         fileUrl: t.exposeString('fileUrl'),
+        previewImage: t.relation('previewImage', { nullable: true }),
         createdAt: t.expose('createdAt', { type: 'DateTime' }),
         updatedAt: t.expose('updatedAt', { type: 'DateTime' }),
         owner: t.relation('owner'),
@@ -430,6 +433,45 @@ export class DocumentSchema extends PothosSchema {
           })
         },
       }),
+      updateDocumentPreviewImage: t.prismaField({
+        type: this.document(),
+        args: {
+          documentId: t.arg({ type: 'String', required: true }),
+          imageId: t.arg({ type: 'String', required: true }),
+        },
+        resolve: async (query, _parent, args, ctx: SchemaContext) => {
+          if (ctx.isSubscription) {
+            throw new Error('Not allowed')
+          }
+          const document = await this.prisma.document.findUnique({
+            where: { id: args.documentId },
+            include: {
+              collaborators: true,
+            },
+          })
+          if (!document) {
+            throw new Error('Document not found')
+          }
+          if (
+            document.ownerId !== ctx.http?.me?.id &&
+            !document.collaborators.some((c) => c.userId === ctx.http?.me?.id && (c.writable || c.readable))
+          ) {
+            throw new Error('User is not owner or collaborator of document')
+          }
+          // check if imageId is exist
+          const image = await this.prisma.uploadedFile.findUnique({
+            where: { id: args.imageId },
+          })
+          if (!image) {
+            throw new Error('Image not found')
+          }
+          return await this.prisma.document.update({
+            ...query,
+            where: { id: args.documentId },
+            data: { previewImageId: args.imageId },
+          })
+        },
+      }),
     }))
 
     this.builder.subscriptionFields((t) => ({
@@ -471,6 +513,7 @@ export class DocumentSchema extends PothosSchema {
             `${DocumentEvent.PAGE_CREATED}.${documentId}`,
             `${DocumentEvent.PAGE_DELETED}.${documentId}`,
             `${DocumentEvent.ACTIVE_DOCUMENT_ID_CHANGED}.${documentId}`,
+            `${DocumentEvent.AI_SUGGESTION}.${documentId}`,
           ]) as unknown as AsyncIterable<DocumentDelta>
         },
         resolve: async (payload: DocumentDelta, _args, ctx: SchemaContext) => {
@@ -481,7 +524,10 @@ export class DocumentSchema extends PothosSchema {
             // using randomize sync mechanism to avoid performance issue
             const random = Math.random()
             // 0.5% chance to request sync
-            if (random <= 0.005) {
+            if (random <= 0.05) {
+              // check grammar too
+              this.documentService.checkGrammarForPage(payload.documentId, payload.pageIndex)
+              Logger.log('request sync', 'request sync')
               payload.requestSync = true
               return payload
             }

src/Document/document.service.ts

@@ -2,13 +2,152 @@ import { Inject, Injectable, forwardRef } from '@nestjs/common'
 
 import { Logger } from '@nestjs/common'
 import Delta, { Op } from 'quill-delta'
+import { OpenaiService, PromptType } from 'src/OpenAI/openai.service'
 import { PrismaService } from 'src/Prisma/prisma.service'
+import { PubSubService } from 'src/PubSub/pubsub.service'
+import { RedisService } from 'src/Redis/redis.service'
 import { MinioService } from '../Minio/minio.service'
+import { DocumentEvent } from './document.event'
 import { DocumentDelta } from './document.type'
 @Injectable()
 export class DocumentService {
   constructor(
     private readonly prisma: PrismaService,
     private readonly minio: MinioService,
+    private readonly openai: OpenaiService,
+    private readonly redis: RedisService,
+    private readonly pubSub: PubSubService,
   ) {}
+
+  // page to list paragraph as delta
+  async pageToParagraph(content: Delta): Promise<Delta> {
+    // Create a new Delta to store paragraphs
+    const paragraphDelta = new Delta()
+
+    // Iterate through the operations in the content Delta
+    for (const op of content.ops) {
+      // If the operation is a string insert
+      if (typeof op.insert === 'string') {
+        // Split the text by newline characters
+        const paragraphs = op.insert.split('\n').filter((p) => p.trim() !== '')
+
+        // Add each non-empty paragraph as a separate insert with a newline
+        paragraphs.forEach((paragraph, index) => {
+          paragraphDelta.insert(paragraph)
+
+          // Add a newline after each paragraph except the last one
+          if (index < paragraphs.length - 1) {
+            paragraphDelta.insert('\n')
+          }
+        })
+      } else {
+        // If the operation is not a string (e.g., an embedded object),
+        // add it to the paragraphDelta as-is
+        paragraphDelta.push(op)
+      }
+    }
+
+    return paragraphDelta
+  }
+
+  async paragraphToSentence(content: Delta): Promise<Delta> {
+    // Create a new Delta to store sentences
+    const sentenceDelta = new Delta()
+
+    // Iterate through the operations in the content Delta
+    for (const op of content.ops) {
+      // If the operation is a string insert
+      if (typeof op.insert === 'string') {
+        // Split the text into paragraphs first
+        const paragraphs = op.insert.split('\n').filter((p) => p.trim() !== '')
+
+        paragraphs.forEach((paragraph, paragraphIndex) => {
+          // Split paragraph into sentences using regex
+          // This handles common sentence-ending punctuation: . ! ?
+          const sentences = paragraph.split(/(?<=[.!?])\s+/).filter((s) => s.trim() !== '')
+
+          sentences.forEach((sentence, sentenceIndex) => {
+            sentenceDelta.insert(sentence.trim())
+
+            // Add a newline after each sentence except the last one in the paragraph
+            if (sentenceIndex < sentences.length - 1) {
+              sentenceDelta.insert('\n')
+            }
+          })
+
+          // Add a newline between paragraphs, except after the last paragraph
+          if (paragraphIndex < paragraphs.length - 1) {
+            sentenceDelta.insert('\n')
+          }
+        })
+      } else {
+        // If the operation is not a string (e.g., an embedded object),
+        // add it to the sentenceDelta as-is
+        sentenceDelta.push(op)
+      }
+    }
+
+    return sentenceDelta
+  }
+
+  async pageToSentence(documentId: string, pageId: number): Promise<Delta> {
+    const content = await this.minio.getDocumentContent(documentId, pageId)
+    const paragraphDelta = await this.pageToParagraph(content)
+    const sentenceDelta = await this.paragraphToSentence(paragraphDelta)
+    return sentenceDelta
+  }
+
+  // check grammar for a page by parallely send each sentence to OpenAI service as text and get the result, after that return the result as delta and publish the result to the document
+  async checkGrammarForPage(documentId: string, pageId: number): Promise<void> {
+    const sentenceDelta = await this.pageToSentence(documentId, pageId)
+
+    // Extract the entire page content as a single text
+    const pageText = sentenceDelta.ops
+      .filter((op) => typeof op.insert === 'string')
+      .map((op) => op.insert as string)
+      .join(' ')
+
+    // Create a unique cache key for the entire page
+    const cacheKey = `grammar_check:${documentId}:${pageId}`
+
+    // Try to get cached result first
+    const cachedResult = await this.redis.get(cacheKey)
+
+    if (cachedResult) {
+      Logger.log('Cached grammar check result exists', 'Grammar Check')
+      return
+    }
+
+    // Process the entire page text
+    const grammarCheckResult = await this.openai.processText(pageText, 0, PromptType.CHECK_GRAMMAR)
+    
+    if (!grammarCheckResult.result) {
+      return
+    }
+
+    // Cache the result
+    await this.redis.setPermanent(cacheKey, JSON.stringify(grammarCheckResult))
+
+    // Calculate diff between original page and corrected page
+    const diff = await this.diff(pageText, grammarCheckResult.result)
+
+    // Build payload
+    const payload: DocumentDelta = {
+      documentId,
+      pageIndex: pageId,
+      ops: diff.ops,
+    }
+
+    // Publish the result to the subscriber
+    this.pubSub.publish(`${DocumentEvent.AI_SUGGESTION}.${documentId}.${pageId}`, payload)
+  }
+
+  async diff(original: string, corrected: string): Promise<Delta> {
+    const originalDelta = new Delta().insert(original)
+    const correctedDelta = new Delta().insert(corrected)
+    const diff = originalDelta.diff(correctedDelta)
+    Logger.log(diff, 'diff')
+    return diff
+  }
 }
+

src/Minio/minio.service.ts

@@ -32,6 +32,22 @@ export class MinioService {
     return { result, filename, mimetype, actualFileName }
   }
 
+  async upsertFile(name: string, file: FileUpload, category: string) {
+    const { mimetype, createReadStream, filename: actualFileName } = await file
+
+    const fileBuffer = createReadStream()
+
+    return await this.minioClient.putObject(
+      this.configService.get('BUCKET_NAME') ?? 'epess',
+      `${category}/${name}`,
+      fileBuffer,
+      undefined,
+      {
+        'Content-Type': mimetype,
+      },
+    )
+  }
+
   async getFileUrl(fileName: string, category: string, presignUrl?: string) {
     if (!fileName) {
       return null
@@ -124,6 +140,15 @@ export class MinioService {
       })
     })
   }
+
+  async getDocumentContent(id: string, page: number): Promise<Delta> {
+    const delta = await this.minioClient.getObject(
+      this.configService.get('BUCKET_NAME') ?? 'epess',
+      `documents/${id}/${page}`,
+    )
+    const buffer = await delta.read()
+    return JSON.parse(buffer.toString()) as Delta
+  }
   // export document to docx format by get all pages and convert to docx
   async exportDocument(id: string) {
     // get all pages

src/OpenAI/openai.service.ts

@@ -3,9 +3,44 @@ import { Injectable, Logger } from '@nestjs/common'
 import { OpenAI } from 'openai'
 import { zodResponseFormat } from 'openai/helpers/zod'
 import Delta from 'quill-delta'
-import { z } from 'zod'
 
-const DELTA_INSTRUCTIONS = ''
+export enum PromptType {
+  CHECK_GRAMMAR = 'CHECK_GRAMMAR',
+  REWRITE_TEXT = 'REWRITE_TEXT',
+  SUMMARIZE = 'SUMMARIZE',
+  TRANSLATE = 'TRANSLATE',
+  EXPAND_CONTENT = 'EXPAND_CONTENT',
+}
+const prompts = {
+  CHECK_GRAMMAR: `You will be provided with text delimited by triple quotes.
+  If the text contains spell or grammar errors provide the text corrected.
+  If the text does not contain errors return back the text delimited by triple quotes.
+  \"\"\"{text}\"\"\"
+  `,
+  REWRITE_TEXT: `You will be provided with text delimited by triple quotes.
+  Rewrite the text to improve clarity, conciseness, and overall readability.
+  Maintain the original meaning and tone of the text.
+  If the text is already well-written, return it as is.
+  \"\"\"{text}\"\"\"
+  `,
+  SUMMARIZE: `You will be provided with text delimited by triple quotes.
+  Create a concise summary that captures the key points and main ideas of the text.
+  The summary should be clear, objective, and no longer than 3-4 sentences.
+  \"\"\"{text}\"\"\"
+  `,
+  TRANSLATE: `You will be provided with text delimited by triple quotes and a target language.
+  Translate the text accurately to the specified language, preserving the original tone and meaning.
+  Provide the translation within triple quotes.
+  Language: {language}
+  \"\"\"{text}\"\"\"
+  `,
+  EXPAND_CONTENT: `You will be provided with text delimited by triple quotes.
+  Expand on the given text by adding more details, examples, or explanations.
+  Maintain the original style and intent of the text.
+  Provide additional context that enhances understanding.
+  \"\"\"{text}\"\"\"
+  `,
+}
 
 @Injectable()
 export class OpenaiService {
@@ -24,173 +59,30 @@ export class OpenaiService {
     return response.choices[0].message.content
   }
 
-  async documentSuggestEditDelta(documentDelta: Delta): Promise<Delta> {
-    // Redefine the schema for Delta.Op with more detailed validations
-    const deltaOpSchema = z
-      .object({
-        insert: z.string().optional(),
-        delete: z.number().optional(),
-        retain: z.number().optional(),
-        attributes: z
-          .object({
-            bold: z.boolean().optional(),
-            italic: z.boolean().optional(),
-            color: z.string().optional(),
-            link: z.string().optional(),
-            header: z.number().optional(),
-            // Add any other attributes that may be required here
-          })
-          .optional(),
-      })
-      .strict()
-
-    // Define the overall schema for Delta operations
-    const schema = z.object({
-      ops: z.array(deltaOpSchema),
-    })
-
-    const systemPrompt = `
-    System: You are an assistant tasked with editing document content in QuillJS Delta format. You will receive a document and editing instructions. Your goal is to modify the document while maintaining the exact **language**, **tone**, **structure**, and **intent** of the original content. Every modification must be consistent with the original style. Please follow the instructions meticulously and ensure all operations are precisely represented in Delta format.
-
-1. **Document Content (Delta Format):**
-   - The document content is provided in QuillJS Delta format. This is the **original** document content. Do not alter or change the language or tone of the original content unless specifically instructed.
-
-   ---BEGIN---
-   (Insert the original document content in Delta format here.)
-   ---END---
-
-2. **Editing Instructions (Delta Format):**
-   - Below are the instructions for editing the document. Follow these instructions strictly and ensure that changes are applied precisely as requested. Do not introduce new meanings or alter the context of the original document.
-
-   ---BEGIN---
-   (Insert detailed instructions here, specifying what text should be modified, added, deleted, or reformatted.)
-   ---END---
-
-3. **Editing Requirements:**
-   - **Language Consistency**: The document's language must remain consistent throughout. Do **not** introduce new phrasing, terms, or language that deviates from the original document's style or tone.
-   - **Preserve the Intent**: Ensure that the intended meaning of the document is maintained. If the instructions suggest changes, they must fit within the overall context and tone of the original.
-   - **Formatting and Structure**: Respect the original formatting and structural layout. Any modifications should adhere to the same format as the source document.
-   - **Modification Representation**: All changes should be accurately reflected in **Delta format**, which includes:
-     - **Insert**: For adding new text or content (e.g., images, links).
-     - **Delete**: For removing content.
-     - **Retain**: For keeping content but potentially modifying formatting or attributes.
-
-4. **Web Browsing and Reference Suggestions:**
-   - If applicable, suggest **web references** that are directly relevant to the content. If web browsing is required, ensure the reference links are **reliable**, **valid**, and fit seamlessly within the document's language.
-   - When suggesting external links, make sure they are:
-     - **Relevant** to the document's context.
-     - **Formatted correctly** with proper citations.
-
-5. **Delta Operations (Insert, Delete, Retain):**
-   - Ensure that all modifications are represented by the appropriate **Delta operations**:
-     - **Insert**: Add new content (text, images, etc.).
-     - **Delete**: Remove content.
-     - **Retain**: Keep content with no change, or only adjust its attributes.
-   - All modifications should be **accurately** expressed using these operations.
-
-6. **Handling Non-Text Content (Embeds):**
-   - If there are changes to non-text elements (such as images, links, or videos), ensure they are added correctly using the **insert** operation with appropriate attributes (e.g., \`link\`, \`image\`).
-   - Example:
-     \`\`\`json
-     {
-       "ops": [
-         {
-           "insert": { "image": "https://example.com/image.png" },
-           "attributes": { "link": "https://example.com" }
-         }
-       ]
-     }
-     \`\`\`
-
-7. **Line and Paragraph Formatting:**
-   - If changes involve formatting at the line or paragraph level (e.g., headings, lists, indentation), apply the appropriate **attributes**:
-     - Example for adding a heading:
-     \`\`\`json
-     {
-       "ops": [
-         { "insert": "New Heading" },
-         { "insert": "\\n", "attributes": { "header": 2 } }
-       ]
-     }
-     \`\`\`
-
-8. **Character Count and Delta Operations:**
-   - Ensure that **Delta operations** correctly represent the changes to text length. Track the **character count** and ensure the Delta format reflects both the original and modified content accurately.
-
-9. **Final Output:**
-   - The **final output** must be a valid QuillJS Delta format document that:
-     - Contains the necessary Delta operations (insert, delete, retain).
-     - Reflects all modifications clearly and precisely, without altering the document’s original tone, intent, or structure.
-
-10. **Additional Instructions for Handling Complex Edits:**
-    - If the document requires complex edits (e.g., restructuring, replacing long sections of text), break the edits into smaller steps and ensure each step follows the same language and tone.
-
-
-
-`
-    const userPrompt = `
-User: I need you to edit the following document in QuillJS Delta format according to the instructions below. 
-
-1. **Document Content (Delta Format):**
-   - Here is the document content in QuillJS Delta format. This is the **original** content of the document, and it must be preserved in **language, tone, and structure** unless specifically instructed otherwise.
-
-   ---BEGIN---
-   ${JSON.stringify(documentDelta)}
-   ---END---
-
-2. **Editing Instructions (Delta Format):**
-   - Below are the detailed instructions for editing the content. Follow these instructions exactly as provided, ensuring that any changes made are in line with the original language and tone of the document.
-
-   ---BEGIN---
-   ${DELTA_INSTRUCTIONS}
-   ---END---
-
-3. **Requirements:**
-   - **Language and Tone**: Do not deviate from the original language or tone. Any changes should be phrased in the same style and maintain the same level of formality, diction, and style.
-   - **Preserve Intent**: Ensure that all edits preserve the original intent and meaning of the document. Avoid any rephrasing or changes that alter the message.
-   - **Accurate Delta Representation**: Ensure all changes are accurately reflected in **Delta format**:
-     - **Insert**: For new content.
-     - **Delete**: For removed content.
-     - **Retain**: For unchanged content with potential formatting changes.
-     
-4. **Web Browsing and Reference Suggestions:**
-   - If needed, suggest **web references** that are relevant to the document's context. Only insert valid URLs with descriptions that complement the document's language and tone.
-   
-5. **Output:**
-   - Provide the final **Delta format output** that accurately reflects all changes, including inserts, deletes, and retains, while maintaining consistency in language and tone.
-
-6. **Delta Format Schema:**
-   - Ensure the Delta format adheres to the following structure:
-     - **Insert**: For adding new content (text, links, images, etc.).
-     - **Delete**: For removing content.
-     - **Retain**: For keeping content, but possibly modifying formatting or attributes.
-     
-7. **Handling Non-Text Embeds:**
-   - For any non-text content, such as **images** or **videos**, use the appropriate **insert** operation with attributes. Ensure that any media links are formatted correctly and are relevant to the document’s content.
-`
-    Logger.log(userPrompt, 'userPrompt')
-
+  async processText(text: string, sentenceIndex: number, prompt: PromptType) {
+    const systemPrompt = 'You are an expert proofreader, the most advanced AI tool on the planet.'
+    const userPrompt = prompts[prompt].replace('{text}', text)
     try {
       const response = await this.openai.chat.completions.create({
-        model: 'gpt-4o',
+        model: 'claude-3-haiku-20240307',
         messages: [
           { role: 'system', content: systemPrompt },
           { role: 'user', content: userPrompt },
         ],
-        response_format: zodResponseFormat(schema, 'Delta'),
       })
 
-      const result = JSON.parse(response.choices[0].message.content ?? '') as Delta
-
-      // Validate that the result matches the expected structure
-      schema.parse(result)
-
-      Logger.log(result, 'result')
-      return result
+      const result = response.choices[0].message.content
+      // split to get data only in triple quotes
+      const resultData = result?.split('"""')[1]
+      Logger.log(resultData, 'result')
+      return {
+        sentenceIndex,
+        result: resultData,
+      }
       // biome-ignore lint/suspicious/noExplicitAny: <explanation>
     } catch (error: any) {
-      Logger.error(`Error in Delta processing: ${error.message}`, 'OpenaiService')
-      throw new Error('Failed to process Delta changes.')
+      Logger.error(`Error in text processing: ${error.message}`, 'OpenaiService')
+      throw new Error('Failed to process data')
     }
   }
 }

src/Redis/redis.service.ts

@@ -17,6 +17,9 @@ export class RedisService {
   async set(key: string, value: string, expireAt: number) {
     return await this.redis.set(key, value, 'EXAT', expireAt)
   }
+  async setPermanent(key: string, value: string) {
+    return await this.redis.set(key, value)
+  }
 
   async del(key: string) {
     return await this.redis.del(key)
@@ -38,4 +41,20 @@ export class RedisService {
   async setUser(sessionId: string, user: User, expireAt: number) {
     return await this.set(sessionId, JSON.stringify(user), expireAt)
   }
+
+  async rpush(key: string, value: string) {
+    return await this.redis.rpush(key, value)
+  }
+
+  async lrange(key: string, start: number, end: number) {
+    return await this.redis.lrange(key, start, end)
+  }
+
+  async blpop(key: string, timeout: number) {
+    return await this.redis.blpop(key, timeout)
+  }
+
+  async publish(channel: string, message: string) {
+    return await this.redis.publish(channel, message)
+  }
 }

src/Restful/restful.controller.ts

@@ -7,12 +7,12 @@ import { RestfulService } from './restful.service'
 export class RestfulController {
   constructor(private readonly restfulService: RestfulService) {}
 
-  @Get()
-  @ApiOperation({ summary: 'Get all items' })
-  @ApiResponse({ status: 200, description: 'Returns all items.' })
-  getAllItems() {
-    return this.restfulService.getAllItems()
-  }
+  // @Get()
+  // @ApiOperation({ summary: 'Get all items' })
+  // @ApiResponse({ status: 200, description: 'Returns all items.' })
+  // getAllItems() {
+  //   return this.restfulService.getAllItems()
+  // }
 
   // @Get(':id')
   // @ApiOperation({ summary: 'Get an item by ID' })

src/Restful/restful.service.ts

@@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common'
 
 @Injectable()
 export class RestfulService {
-  getAllItems() {
-    return 'Hello World'
-  }
+  // getAllItems() {
+  //   return 'Hello World'
+  // }
 }