feat: add Qdrant recording workflow

2026-05-27 10:49:05 +09:00
parent b1689faab7
commit c7e55cfd24
6 changed files with 174 additions and 0 deletions
@@ -0,0 +1,114 @@
+name: Record to Qdrant
+
+on:
+  push:
+    branches: ['**']
+  pull_request:
+    types: [opened, closed, synchronize]
+  create:
+  delete:
+  release:
+    types: [published]
+  workflow_dispatch:
+  schedule:
+    - cron: '0 15 * * *'
+
+jobs:
+  record-event:
+    if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    container:
+      image: python:3.11-slim
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Install dependencies
+        run: apt-get update -qq && apt-get install -y -qq curl bash git
+
+      - name: Checkout
+        run: |
+          git clone --depth 1 http://admin:0ebeb908eb6bc62c20d4d0cb937970651859f409@10.43.165.66:3000/k3s-manifests/pod-manifests.git /tmp/repo
+
+      - name: Record event to Qdrant
+        env:
+          EVENT: ${{ github.event_name }}
+          REPO: ${{ github.repository }}
+          BRANCH: ${{ github.ref_name }}
+          ACTOR: ${{ github.actor }}
+          RUN_ID: ${{ github.run_id }}
+          COMMIT: ${{ github.sha }}
+        run: |
+          COMMIT_SHORT=$(echo "${COMMIT}" | cut -c1-8)
+          TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+          CONTENT="[${EVENT}] ${REPO} @ ${BRANCH} by ${ACTOR} | commit: ${COMMIT_SHORT} | run: ${RUN_ID} | time: ${TIMESTAMP}"
+          echo "Content: ${CONTENT}"
+          echo -n "${CONTENT}" > /tmp/content.txt
+
+          python3 -c "import json;open('/tmp/embed_req.json','w').write(json.dumps({'model':'bge-m3:latest','input':open('/tmp/content.txt').read()}))"
+
+          curl -s -X POST http://192.168.11.124:11434/api/embed \
+            -H "Content-Type: application/json" \
+            --data-binary @/tmp/embed_req.json > /tmp/embed_response.json
+
+          export TIMESTAMP RUN_ID EVENT REPO BRANCH ACTOR
+          python3 /tmp/repo/scripts/record_event.py
+
+          curl -s -X PUT http://192.168.11.124:6333/collections/gitea_actions/points \
+            -H "Content-Type: application/json" \
+            --data-binary @/tmp/payload.json
+          echo "Done"
+
+  nightly-digest:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    container:
+      image: python:3.11-slim
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Install dependencies
+        run: apt-get update -qq && apt-get install -y -qq curl bash git
+
+      - name: Checkout
+        run: |
+          git clone --depth 1 http://admin:0ebeb908eb6bc62c20d4d0cb937970651859f409@10.43.165.66:3000/k3s-manifests/pod-manifests.git /tmp/repo
+
+      - name: Summarize and store digest
+        run: |
+          TODAY=$(date -u +"%Y-%m-%d")
+          TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+
+          curl -s -X POST http://192.168.11.124:6333/collections/gitea_actions/points/scroll \
+            -H "Content-Type: application/json" \
+            --data-binary "{\"limit\":100,\"with_payload\":true,\"with_vector\":false,\"filter\":{\"must\":[{\"key\":\"timestamp\",\"range\":{\"gte\":\"${TODAY}T00:00:00Z\"}}]}}" \
+            > /tmp/scroll.json
+
+          COUNT=$(python3 -c "import json;d=json.load(open('/tmp/scroll.json'));print(len(d.get('result',{}).get('points',[])))")
+          echo "Records today: ${COUNT}"
+          if [ "${COUNT}" = "0" ]; then echo "No records, skipping."; exit 0; fi
+
+          python3 /tmp/repo/scripts/extract_texts.py
+
+          TODAY=${TODAY} python3 /tmp/repo/scripts/build_gen_payload.py
+
+          curl -s -X POST http://192.168.11.124:11434/api/generate \
+            -H "Content-Type: application/json" \
+            --data-binary @/tmp/gen_payload.json > /tmp/gen_response.json
+
+          python3 -c "import json;open('/tmp/summary.txt','w').write(json.load(open('/tmp/gen_response.json')).get('response',''))"
+          echo "Summary: $(cat /tmp/summary.txt | head -c 100)"
+
+          python3 /tmp/repo/scripts/build_embed_input.py
+
+          curl -s -X POST http://192.168.11.124:11434/api/embed \
+            -H "Content-Type: application/json" \
+            --data-binary @/tmp/embed_input.json > /tmp/embed_response.json
+
+          TODAY=${TODAY} TIMESTAMP=${TIMESTAMP} COUNT=${COUNT} python3 /tmp/repo/scripts/build_digest_payload.py
+
+          RESULT=$(curl -s -X PUT http://192.168.11.124:6333/collections/llm_conversations_digest/points \
+            -H "Content-Type: application/json" \
+            --data-binary @/tmp/digest_payload.json)
+          echo "Digest result: ${RESULT}"