feat: add Qdrant recording workflow

.gitea/workflows/record-to-qdrant.yml

@@ -0,0 +1,114 @@
+name: Record to Qdrant
+
+on:
+  push:
+    branches: ['**']
+  pull_request:
+    types: [opened, closed, synchronize]
+  create:
+  delete:
+  release:
+    types: [published]
+  workflow_dispatch:
+  schedule:
+    - cron: '0 15 * * *'
+
+jobs:
+  record-event:
+    if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    container:
+      image: python:3.11-slim
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Install dependencies
+        run: apt-get update -qq && apt-get install -y -qq curl bash git
+
+      - name: Checkout
+        run: |
+          git clone --depth 1 http://admin:0ebeb908eb6bc62c20d4d0cb937970651859f409@10.43.165.66:3000/k3s-manifests/pod-manifests.git /tmp/repo
+
+      - name: Record event to Qdrant
+        env:
+          EVENT: ${{ github.event_name }}
+          REPO: ${{ github.repository }}
+          BRANCH: ${{ github.ref_name }}
+          ACTOR: ${{ github.actor }}
+          RUN_ID: ${{ github.run_id }}
+          COMMIT: ${{ github.sha }}
+        run: |
+          COMMIT_SHORT=$(echo "${COMMIT}" | cut -c1-8)
+          TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+          CONTENT="[${EVENT}] ${REPO} @ ${BRANCH} by ${ACTOR} | commit: ${COMMIT_SHORT} | run: ${RUN_ID} | time: ${TIMESTAMP}"
+          echo "Content: ${CONTENT}"
+          echo -n "${CONTENT}" > /tmp/content.txt
+
+          python3 -c "import json;open('/tmp/embed_req.json','w').write(json.dumps({'model':'bge-m3:latest','input':open('/tmp/content.txt').read()}))"
+
+          curl -s -X POST http://192.168.11.124:11434/api/embed \
+            -H "Content-Type: application/json" \
+            --data-binary @/tmp/embed_req.json > /tmp/embed_response.json
+
+          export TIMESTAMP RUN_ID EVENT REPO BRANCH ACTOR
+          python3 /tmp/repo/scripts/record_event.py
+
+          curl -s -X PUT http://192.168.11.124:6333/collections/gitea_actions/points \
+            -H "Content-Type: application/json" \
+            --data-binary @/tmp/payload.json
+          echo "Done"
+
+  nightly-digest:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    container:
+      image: python:3.11-slim
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Install dependencies
+        run: apt-get update -qq && apt-get install -y -qq curl bash git
+
+      - name: Checkout
+        run: |
+          git clone --depth 1 http://admin:0ebeb908eb6bc62c20d4d0cb937970651859f409@10.43.165.66:3000/k3s-manifests/pod-manifests.git /tmp/repo
+
+      - name: Summarize and store digest
+        run: |
+          TODAY=$(date -u +"%Y-%m-%d")
+          TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+
+          curl -s -X POST http://192.168.11.124:6333/collections/gitea_actions/points/scroll \
+            -H "Content-Type: application/json" \
+            --data-binary "{\"limit\":100,\"with_payload\":true,\"with_vector\":false,\"filter\":{\"must\":[{\"key\":\"timestamp\",\"range\":{\"gte\":\"${TODAY}T00:00:00Z\"}}]}}" \
+            > /tmp/scroll.json
+
+          COUNT=$(python3 -c "import json;d=json.load(open('/tmp/scroll.json'));print(len(d.get('result',{}).get('points',[])))")
+          echo "Records today: ${COUNT}"
+          if [ "${COUNT}" = "0" ]; then echo "No records, skipping."; exit 0; fi
+
+          python3 /tmp/repo/scripts/extract_texts.py
+
+          TODAY=${TODAY} python3 /tmp/repo/scripts/build_gen_payload.py
+
+          curl -s -X POST http://192.168.11.124:11434/api/generate \
+            -H "Content-Type: application/json" \
+            --data-binary @/tmp/gen_payload.json > /tmp/gen_response.json
+
+          python3 -c "import json;open('/tmp/summary.txt','w').write(json.load(open('/tmp/gen_response.json')).get('response',''))"
+          echo "Summary: $(cat /tmp/summary.txt | head -c 100)"
+
+          python3 /tmp/repo/scripts/build_embed_input.py
+
+          curl -s -X POST http://192.168.11.124:11434/api/embed \
+            -H "Content-Type: application/json" \
+            --data-binary @/tmp/embed_input.json > /tmp/embed_response.json
+
+          TODAY=${TODAY} TIMESTAMP=${TIMESTAMP} COUNT=${COUNT} python3 /tmp/repo/scripts/build_digest_payload.py
+
+          RESULT=$(curl -s -X PUT http://192.168.11.124:6333/collections/llm_conversations_digest/points \
+            -H "Content-Type: application/json" \
+            --data-binary @/tmp/digest_payload.json)
+          echo "Digest result: ${RESULT}"

scripts/build_digest_payload.py

@@ -0,0 +1,22 @@
+import json, uuid, os
+
+summary = open('/tmp/summary.txt').read()
+vector = json.load(open('/tmp/embed_response.json'))['embeddings'][0]
+point_id = str(uuid.uuid4())
+
+payload = {
+    "points": [{
+        "id": point_id,
+        "vector": vector,
+        "payload": {
+            "date": os.environ.get('TODAY', ''),
+            "type": "gitea_daily_digest",
+            "record_count": int(os.environ.get('COUNT', '0')),
+            "timestamp": os.environ.get('TIMESTAMP', ''),
+            "content": summary
+        }
+    }]
+}
+with open('/tmp/digest_payload.json', 'w') as f:
+    json.dump(payload, f)
+print(f"Digest Point ID: {point_id}")

scripts/build_embed_input.py

@@ -0,0 +1,3 @@
+import json
+summary = open('/tmp/summary.txt').read()
+open('/tmp/embed_input.json','w').write(json.dumps({'model':'bge-m3:latest','input':summary}))

scripts/build_gen_payload.py

@@ -0,0 +1,6 @@
+import json, sys, os
+today = os.environ.get('TODAY', '')
+prompt = open('/tmp/texts.txt').read()
+full_prompt = f"以下は{today}のGitea操作ログです。重要な変更点と傾向を日本語300字以内でまとめてください:\n{prompt}"
+payload = {'model': 'qwen2.5:7b', 'prompt': full_prompt, 'stream': False}
+open('/tmp/gen_payload.json','w').write(json.dumps(payload))

scripts/extract_texts.py

@@ -0,0 +1,4 @@
+import json
+d = json.load(open('/tmp/scroll.json'))
+texts = '\n'.join(p.get('payload',{}).get('content','') for p in d.get('result',{}).get('points',[]))
+open('/tmp/texts.txt','w').write(texts)

scripts/record_event.py

@@ -0,0 +1,25 @@
+import json, uuid, sys, os
+
+content = open('/tmp/content.txt').read()
+embed_resp = json.load(open('/tmp/embed_response.json'))
+vector = embed_resp['embeddings'][0]
+point_id = str(uuid.uuid4())
+
+payload = {
+    "points": [{
+        "id": point_id,
+        "vector": vector,
+        "payload": {
+            "run_id": os.environ.get('RUN_ID', ''),
+            "event": os.environ.get('EVENT', ''),
+            "repository": os.environ.get('REPO', ''),
+            "branch": os.environ.get('BRANCH', ''),
+            "actor": os.environ.get('ACTOR', ''),
+            "timestamp": os.environ.get('TIMESTAMP', ''),
+            "content": content
+        }
+    }]
+}
+with open('/tmp/payload.json', 'w') as f:
+    json.dump(payload, f)
+print(f"Point ID: {point_id}")