adjusted llm query code

jupyter_notebooks/.ipynb_checkpoints/Section_2-3-1_Tag_occurences-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

jupyter_notebooks/.ipynb_checkpoints/Section_2-3-4_Figure_8_Step_1_LLM_annotation-checkpoint.ipynb

@@ -466,7 +466,7 @@
     "            'max_new_tokens': 512,\n",
     "            'temperature': 0.05,\n",
     "            'do_sample': True,\n",
-    "            'top_p': 0.8,\n",
+    "            'top_p': 1.0,\n",
     "        }\n",
     "    },\n",
     "    'gemma': {\n",
@@ -477,7 +477,7 @@
     "            'max_new_tokens': 512,\n",
     "            'temperature': 0.1,\n",
     "            'do_sample': True,\n",
-    "            'top_p': 0.9,\n",
+    "            'top_p': 1.0,\n",
     "        }\n",
     "    },\n",
     "    'qwen': {\n",
@@ -489,7 +489,7 @@
     "            llm_int8_has_fp16_weight=False\n",
     "        ),\n",
     "        'generation_params': {\n",
-    "            'max_new_tokens': 100,\n",
+    "            'max_new_tokens': 512,\n",
     "            'temperature': 0.1,\n",
     "            'do_sample': False,\n",
     "        }\n",
@@ -951,6 +951,14 @@
     "# annotate_dataset(model_type='mistral', test_mode=True, test_size=100)\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "6431d347-d80c-4e8b-83a7-531e5df95a72",
+   "metadata": {},
+   "source": [
+    "## EuroLLM-9B-Instruct"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -987,7 +995,7 @@
     "output_file = current_dir.parent / f\"data/CSV/eurollm_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
     "\n",
     "# Model settings\n",
-    "MODEL_NAME = \"utter-project/EuroLLM-9B\"\n",
+    "MODEL_NAME = \"utter-project/EuroLLM-9B-Instruct\"\n",
     "#MODEL_NAME = \"Qwen/Qwen2.5-32B-Instruct\"\n",
     "#MODEL_NAME = \"Qwen/Qwen2.5-14B-Instruct\"\n",
     "#MODEL_NAME =  \"Qwen/Qwen3-235B-A22B-Instruct-2507-FP8\"\n",
@@ -1159,27 +1167,19 @@
     "3. Gender: MUST be exactly one word: Male, Female, Other, or Unknown. VALUE ONLY.\n",
     "4. Professions: Choose up to 3 from this list ONLY: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality. Comma-separated. VALUE ONLY.\n",
     "5. Primary country: Country name only (e.g., \"China\", \"United States\", \"Colombia\"). VALUE ONLY.\n",
-    "6. If uncertain about an item, write \"Unknown\"\n",
     "\n",
     "CRITICAL RULES FOR PROFESSIONS (Line 4):\n",
     "- ONLY use the exact profession categories listed above\n",
     "- DO NOT use descriptive words like \"sexy\", \"photorealistic\", \"celebrity\"\n",
     "- DO NOT copy the hint words as professions\n",
-    "- If uncertain write \"Unknown\"\n",
+    "- If uncertain about profession, write \"Unknown\"\n",
     "- Valid professions are ONLY: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality\n",
     "- Actress = actor, streamer = online personality, YouTuber = online personality\n",
     "\n",
     "OTHER RULES:\n",
     "- Use \"Unknown\" when uncertain or for fictional characters\n",
     "- NO explanatory text, NO labels like \"Gender:\", NO prefixes\n",
-    "- For multi-role people, list up to 3 categories by relevance\n",
-    "\n",
-    "EXAMPLE FORMAT:\n",
-    "1. Taylor Swift\n",
-    "2. None\n",
-    "3. Female\n",
-    "4. singer/musician, public figure\n",
-    "5. United States\"\"\"\n",
+    "- For multi-role people, list up to 3 categories by relevance\"\"\"\n",
     "\n",
     "# Create prompts\n",
     "print(\"\\nCreating prompts...\")\n",
@@ -1418,10 +1418,500 @@
     "print(\"✅ Finished full dataset.\")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "472e5ac2-ec04-4bfa-8a67-116277238c15",
+   "metadata": {},
+   "source": [
+    "## Mistral 24b instruct"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "a55a5e30-83f3-4f7c-a537-b1216d4e8a07",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-12-09T22:16:21.002786Z",
+     "iopub.status.busy": "2025-12-09T22:16:21.002337Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading model: mistralai/Mistral-Small-Instruct-2409\n",
+      "Cache directory: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/models\n",
+      "This may take a while on first run (~65GB download)...\n",
+      "\n",
+      "Device: cuda\n",
+      "Loading tokenizer...\n",
+      "✅ Tokenizer loaded\n",
+      "Loading model (this may take several minutes)...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "`torch_dtype` is deprecated! Use `dtype` instead!\n",
+      "Loading checkpoint shards: 100%|██████████| 9/9 [02:42<00:00, 18.06s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Model loaded\n",
+      "VRAM used: 21.40 GB\n",
+      "\n",
+      "Loading raw input CSV...\n",
+      "Loaded 50861 rows from input file\n",
+      "Found existing annotations, merging...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/tmp/ipykernel_3104208/1997558719.py:113: DtypeWarning: Columns (52,53,54,55,56) have mixed types. Specify dtype option on import or set low_memory=False.\n",
+      "  existing_df = pd.read_csv(output_file)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Existing annotations has 50861 rows\n",
+      "Merged annotations, continuing with 50861 total rows\n",
+      "✅ Loaded professions.csv\n",
+      "✅ Loaded profession mapping with 9 categories\n",
+      "Loaded 50861 rows\n",
+      "\n",
+      "Profession categories (9):\n",
+      "  - actor\n",
+      "  - adult performer\n",
+      "  - singer/musician\n",
+      "  - model\n",
+      "  - online personality\n",
+      "  - public figure\n",
+      "  - voice actor/ASMR\n",
+      "  - sports professional\n",
+      "  - tv personality\n",
+      "\n",
+      "Creating prompts...\n",
+      "✅ Prompts created\n",
+      "Resuming from index 8810\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 0/42051 [00:00<?, ?it/s]/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/bitsandbytes/autograd/_functions.py:181: UserWarning: MatMul8bitLt: inputs will be cast from torch.bfloat16 to float16 during quantization\n",
+      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n",
+      "Mistral Local:   0%|          | 7/42051 [00:57<93:01:03,  7.96s/it] "
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import json\n",
+    "import time\n",
+    "import re\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm\n",
+    "import torch\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
+    "professions_file = current_dir.parent / \"misc/lists/professions.csv\"\n",
+    "professions_mapped_file = current_dir.parent / \"misc/lists/professions_mapped.csv\"\n",
+    "# === PROCESS DATA ===\n",
+    "\n",
+    "\n",
+    "# === CONFIGURATION ===\n",
+    "TEST_MODE = False\n",
+    "TEST_SIZE = 100\n",
+    "MAX_ROWS = 50862\n",
+    "SAVE_INTERVAL = 10\n",
+    "\n",
+    "output_file = current_dir.parent / f\"data/CSV/mistral24_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
+    "index_file = current_dir.parent / \"misc/query_indicies/mistral24_local_query_index.txt\"\n",
+    "\n",
+    "\n",
+    "# Model settings\n",
+    "#MODEL_NAME = \"mistralai/Mistral-Small-3.1-24B-Instruct-2503\"\n",
+    "MODEL_NAME = \"mistralai/Mistral-Small-Instruct-2409\"\n",
+    "#MODEL_NAME = \"mistralai/Mistral-7B-Instruct-v0.3\"\n",
+    "CACHE_DIR = current_dir.parent / \"data/models\"\n",
+    "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Define the SPECIFIC profession categories\n",
+    "PROFESSION_CATEGORIES = [\n",
+    "    \"actor\",\n",
+    "    \"adult performer\",\n",
+    "    \"singer/musician\",\n",
+    "    \"model\",\n",
+    "    \"online personality\",\n",
+    "    \"public figure\",\n",
+    "    \"voice actor/ASMR\",\n",
+    "    \"sports professional\",\n",
+    "    \"tv personality\"\n",
+    "]\n",
+    "\n",
+    "# === LOAD MODEL ===\n",
+    "print(f\"Loading model: {MODEL_NAME}\")\n",
+    "print(f\"Cache directory: {CACHE_DIR}\")\n",
+    "print(f\"This may take a while on first run (~65GB download)...\\n\")\n",
+    "\n",
+    "# Check GPU availability\n",
+    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "print(f\"Device: {device}\")\n",
+    "\n",
+    "if device == \"cpu\":\n",
+    "    print(\"⚠️  WARNING: No GPU detected! Inference will be VERY slow.\")\n",
+    "    print(\"   Consider using a GPU or reducing model size.\")\n",
+    "\n",
+    "# Load tokenizer\n",
+    "print(\"Loading tokenizer...\")\n",
+    "try:\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(\n",
+    "        MODEL_NAME,\n",
+    "        cache_dir=str(CACHE_DIR),\n",
+    "        use_fast=True\n",
+    "    )\n",
+    "except Exception as e:\n",
+    "    print(f\"Failed with use_fast=True, trying use_fast=False...\")\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(\n",
+    "        MODEL_NAME,\n",
+    "        cache_dir=str(CACHE_DIR),\n",
+    "        use_fast=False\n",
+    "    )\n",
+    "\n",
+    "# Ensure pad token is set\n",
+    "if tokenizer.pad_token is None:\n",
+    "    tokenizer.pad_token = tokenizer.eos_token\n",
+    "\n",
+    "print(\"✅ Tokenizer loaded\")\n",
+    "\n",
+    "quantization_config = BitsAndBytesConfig(\n",
+    "    load_in_8bit=True\n",
+    ")\n",
+    "\n",
+    "\n",
+    "# Load model with optimizations\n",
+    "print(\"Loading model (this may take several minutes)...\")\n",
+    "model = AutoModelForCausalLM.from_pretrained(\n",
+    "    MODEL_NAME,\n",
+    "    cache_dir=str(CACHE_DIR),\n",
+    "    torch_dtype=torch.bfloat16,\n",
+    "    quantization_config=quantization_config,\n",
+    "    device_map=\"auto\",\n",
+    "    trust_remote_code=False\n",
+    ")\n",
+    "model.eval()\n",
+    "print(\"✅ Model loaded\")\n",
+    "\n",
+    "# Check VRAM usage\n",
+    "if torch.cuda.is_available():\n",
+    "    vram_gb = torch.cuda.max_memory_allocated() / 1024**3\n",
+    "    print(f\"VRAM used: {vram_gb:.2f} GB\\n\")\n",
+    "\n",
+    "# === LOAD DATA ===\n",
+    "print(\"Loading raw input CSV...\")\n",
+    "df = pd.read_csv(input_file)  # ALWAYS load the full input\n",
+    "print(f\"Loaded {len(df)} rows from input file\")\n",
+    "\n",
+    "# If we have previous annotations, merge them\n",
+    "if output_file.exists():\n",
+    "    print(\"Found existing annotations, merging...\")\n",
+    "    existing_df = pd.read_csv(output_file)\n",
+    "    print(f\"Existing annotations has {len(existing_df)} rows\")\n",
+    "    \n",
+    "    # Update df with existing annotations\n",
+    "    # Only update the columns that were annotated\n",
+    "    annotation_cols = ['full_name', 'aliases', 'gender', 'profession_llm', 'country']\n",
+    "    for col in annotation_cols:\n",
+    "        if col in existing_df.columns:\n",
+    "            df[col] = existing_df[col][:len(df)]  # Make sure we don't exceed df length\n",
+    "    \n",
+    "    print(f\"Merged annotations, continuing with {len(df)} total rows\")\n",
+    "\n",
+    "\n",
+    "# Try to load profession mapping files\n",
+    "try:\n",
+    "    professions_df = pd.read_csv(professions_file)\n",
+    "    print(f\"✅ Loaded professions.csv\")\n",
+    "except:\n",
+    "    print(\"⚠️ Warning: professions.csv not found\")\n",
+    "\n",
+    "try:\n",
+    "    prof_mapped_df = pd.read_csv(professions_mapped_file)\n",
+    "    print(f\"✅ Loaded profession mapping with {len(prof_mapped_df)} categories\")\n",
+    "except:\n",
+    "    print(\"⚠️ Warning: professions_mapped.csv not found, using default categories\")\n",
+    "\n",
+    "profession_str = \", \".join(PROFESSION_CATEGORIES)\n",
+    "\n",
+    "print(f\"Loaded {len(df)} rows\")\n",
+    "print(f\"\\nProfession categories ({len(PROFESSION_CATEGORIES)}):\")\n",
+    "for cat in PROFESSION_CATEGORIES:\n",
+    "    print(f\"  - {cat}\")\n",
+    "\n",
+    "if TEST_MODE:\n",
+    "    print(f\"\\nRunning in TEST MODE with {TEST_SIZE} samples\")\n",
+    "    df = df.head(TEST_SIZE).copy()\n",
+    "elif MAX_ROWS:\n",
+    "    df = df.head(MAX_ROWS).copy()\n",
+    "\n",
+    "# === CREATE PROMPTS (DEEPSEEK STYLE) ===\n",
+    "def create_prompt(row):\n",
+    "    \"\"\"Create prompt for Mistral annotation with specific profession categories.\"\"\"\n",
+    "    name = row['real_name'] if pd.notna(row.get('real_name')) else row.get('name', '')\n",
+    "    \n",
+    "    # Gather hints\n",
+    "    hints = []\n",
+    "    if pd.notna(row.get('likely_profession')):\n",
+    "        hints.append(str(row['likely_profession']))\n",
+    "    if pd.notna(row.get('likely_nationality')):\n",
+    "        hints.append(str(row['likely_nationality']))\n",
+    "    if pd.notna(row.get('likely_country')):\n",
+    "        hints.append(str(row['likely_country']))\n",
+    "    \n",
+    "    # Add tags if we don't have enough hints\n",
+    "    if len(hints) < 3:\n",
+    "        for i in range(1, 8):\n",
+    "            tag_col = f'tag_{i}'\n",
+    "            if tag_col in row and pd.notna(row[tag_col]):\n",
+    "                tag_val = str(row[tag_col])\n",
+    "                if tag_val not in hints:\n",
+    "                    hints.append(tag_val)\n",
+    "                if len(hints) >= 5:\n",
+    "                    break\n",
+    "    \n",
+    "    hint_text = \", \".join(hints[:5]) if hints else \"none\"\n",
+    "    \n",
+    "    return f\"\"\"Given '{name}' ({hint_text}), provide:\n",
+    "1. Full legal name (Western order if non-latin script)\n",
+    "2. Any stage names/aliases (comma separated)\n",
+    "3. Gender (Male/Female/Other/Unknown)\n",
+    "4. Top 3 most likely professions from ONLY these categories:\n",
+    "   - actor\n",
+    "   - adult performer\n",
+    "   - singer/musician\n",
+    "   - model\n",
+    "   - online personality (includes streamers, cosplayers, influencers)\n",
+    "   - public figure (includes politicians, activists, journalists, authors)\n",
+    "   - voice actor/ASMR\n",
+    "   - sports professional\n",
+    "   - tv personality (includes hosts, presenters, reality TV)\n",
+    "\n",
+    "5. Primary country associated\n",
+    "\n",
+    "IMPORTANT:\n",
+    "- Choose professions ONLY from the 9 categories above\n",
+    "- Provide up to 3 professions, comma-separated, ordered by relevance\n",
+    "- Be SPECIFIC: choose the most accurate category for each role\n",
+    "- \"online personality\" includes: streamers, cosplayers, YouTubers, influencers, content creators\n",
+    "- Use 'Unknown' when uncertain or for fictional characters/places\n",
+    "- For multi-role people, list all relevant categories (e.g., \"actor, singer/musician, online personality\")\n",
+    "- For country respond with one word only, for example China or Columbia\n",
+    "- actress = actor\n",
+    "\n",
+    "Respond with exactly 5 numbered lines.\"\"\"\n",
+    "\n",
+    "# Create prompts\n",
+    "print(\"\\nCreating prompts...\")\n",
+    "df['prompt'] = df.apply(create_prompt, axis=1)\n",
+    "print(\"✅ Prompts created\")\n",
+    "\n",
+    "# === QUERY MISTRAL LOCAL ===\n",
+    "def query_mistral_local(prompt: str) -> str:\n",
+    "    \"\"\"Query Mistral locally via transformers.\"\"\"\n",
+    "    try:\n",
+    "        # Format as chat message for Mistral\n",
+    "        messages = [\n",
+    "            {\"role\": \"system\", \"content\": \"You are an assistant that extracts key data on a person based on the name. Respond with exactly 5 numbered lines. For professions, choose ONLY from these categories: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality.\"},\n",
+    "            {\"role\": \"user\", \"content\": prompt}\n",
+    "        ]\n",
+    "        \n",
+    "        # Tokenize\n",
+    "        if hasattr(tokenizer, 'apply_chat_template'):\n",
+    "            text = tokenizer.apply_chat_template(\n",
+    "                messages,\n",
+    "                tokenize=False,\n",
+    "                add_generation_prompt=True\n",
+    "            )\n",
+    "        else:\n",
+    "            # Fallback for older tokenizers\n",
+    "            text = f\"[INST] {prompt} [/INST]\"\n",
+    "        \n",
+    "        inputs = tokenizer([text], return_tensors=\"pt\", padding=True).to(device)\n",
+    "        \n",
+    "        # Generate\n",
+    "        with torch.no_grad():\n",
+    "            outputs = model.generate(\n",
+    "                **inputs,\n",
+    "                max_new_tokens=512,\n",
+    "                temperature=0.05,\n",
+    "                do_sample=True,\n",
+    "                top_p=0.8,\n",
+    "                pad_token_id=tokenizer.pad_token_id if tokenizer.pad_token_id else tokenizer.eos_token_id\n",
+    "            )\n",
+    "        \n",
+    "        # Decode\n",
+    "        generated_ids = outputs[0][inputs['input_ids'].shape[1]:]\n",
+    "        response = tokenizer.decode(generated_ids, skip_special_tokens=True)\n",
+    "        \n",
+    "        return response.strip()\n",
+    "        \n",
+    "    except Exception as e:\n",
+    "        print(f\"Generation error: {e}\")\n",
+    "        return None\n",
+    "\n",
+    "# === PARSE RESPONSE (DEEPSEEK STYLE) ===\n",
+    "def parse_response(response):\n",
+    "    \"\"\"Parse Mistral response into structured fields.\"\"\"\n",
+    "    if not response:\n",
+    "        return {\n",
+    "            'full_name': 'Unknown',\n",
+    "            'aliases': 'Unknown',\n",
+    "            'gender': 'Unknown',\n",
+    "            'profession_llm': 'Unknown',\n",
+    "            'country': 'Unknown'\n",
+    "        }\n",
+    "    \n",
+    "    # Split into lines and clean\n",
+    "    lines = [line.strip() for line in response.split('\\n') if line.strip()]\n",
+    "    \n",
+    "    # Initialize with Unknown values\n",
+    "    fields = {\n",
+    "        'full_name': 'Unknown',\n",
+    "        'aliases': 'Unknown',\n",
+    "        'gender': 'Unknown',\n",
+    "        'profession_llm': 'Unknown',\n",
+    "        'country': 'Unknown'\n",
+    "    }\n",
+    "    \n",
+    "    # Extract information from each numbered line\n",
+    "    for line in lines:\n",
+    "        if line.startswith('1.'):\n",
+    "            fields['full_name'] = line[2:].strip()\n",
+    "        elif line.startswith('2.'):\n",
+    "            fields['aliases'] = line[2:].strip()\n",
+    "        elif line.startswith('3.'):\n",
+    "            fields['gender'] = line[2:].strip()\n",
+    "        elif line.startswith('4.'):\n",
+    "            fields['profession_llm'] = line[2:].strip()\n",
+    "        elif line.startswith('5.'):\n",
+    "            fields['country'] = line[2:].strip()\n",
+    "    \n",
+    "    return fields\n",
+    "\n",
+    "# === PROCESS DATA ===\n",
+    "output_file = current_dir.parent / f\"data/CSV/mistral24_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
+    "index_file = current_dir.parent / \"misc/query_indicies/mistral24_local_query_index.txt\"\n",
+    "\n",
+    "index_file.parent.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Load index\n",
+    "current_index = 0\n",
+    "if index_file.exists():\n",
+    "    try:\n",
+    "        current_index = int(index_file.read_text().strip())\n",
+    "    except:\n",
+    "        current_index = 0\n",
+    "\n",
+    "print(f\"Resuming from index {current_index}\")\n",
+    "\n",
+    "start_time = time.time()\n",
+    "\n",
+    "for i in tqdm(range(current_index, len(df)), desc=\"Mistral Local\"):\n",
+    "\n",
+    "    prompt = df.at[i, \"prompt\"]\n",
+    "\n",
+    "    # -------- MODEL QUERY WITH RETRIES --------\n",
+    "    response = None\n",
+    "    for attempt in range(3):\n",
+    "        response = query_mistral_local(prompt)\n",
+    "        \n",
+    "        # Valid response?\n",
+    "        if response and len(response.strip()) > 10:\n",
+    "            break\n",
+    "        \n",
+    "        print(f\"⚠️ Row {i}: Empty or invalid response, retry {attempt+1}/3\")\n",
+    "        time.sleep(0.5)\n",
+    "\n",
+    "    # If still invalid → DO NOT overwrite previous data\n",
+    "    if not response or len(response.strip()) <= 10:\n",
+    "        print(f\"❌ Row {i}: failed after retries, not writing, not advancing index\")\n",
+    "        continue\n",
+    "\n",
+    "    parsed = parse_response(response)\n",
+    "\n",
+    "    # Additional safety: skip rows that parsed as all 'Unknown'\n",
+    "    if all(v == \"Unknown\" for v in parsed.values()):\n",
+    "        print(f\"❌ Row {i}: parsed as all Unknown (likely model crash); skipping.\")\n",
+    "        continue\n",
+    "\n",
+    "    # -------- WRITE PARSED FIELDS SAFELY --------\n",
+    "    for key, value in parsed.items():\n",
+    "        df.at[i, key] = value\n",
+    "\n",
+    "    # Advance progress ONLY after successful write\n",
+    "    current_index = i + 1\n",
+    "\n",
+    "    # -------- GPU MEMORY CLEANUP --------\n",
+    "    if torch.cuda.is_available():\n",
+    "        torch.cuda.empty_cache()\n",
+    "        torch.cuda.synchronize()\n",
+    "\n",
+    "    # -------- SAVE LIKE YOUR DEEPSEEK VERSION --------\n",
+    "    if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
+    "        df.to_csv(output_file, index=False)\n",
+    "        with open(index_file, \"w\") as f:\n",
+    "            f.write(str(current_index))\n",
+    "        print(f\"💾 Progress saved after row {i+1}\")\n",
+    "\n",
+    "# Final save\n",
+    "df.to_csv(output_file, index=False)\n",
+    "index_file.write_text(str(current_index))\n",
+    "print(\"✅ Finished full dataset.\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d7212e75-0ff6-45a0-8695-c4a3d3e02818",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import transformers\n",
+    "print(f\"Transformers version: {transformers.__version__}\")\n",
+    "\n",
+    "# Check if Mistral3 is available\n",
+    "try:\n",
+    "    from transformers import Mistral3ForCausalLM\n",
+    "    print(\"✅ Mistral3 is available\")\n",
+    "except ImportError:\n",
+    "    print(\"❌ Mistral3 not available in this transformers version\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a6ab032e-246e-4c4e-9776-ff0bfbf6fd9c",
    "metadata": {},
    "outputs": [],
    "source": []

jupyter_notebooks/Section_2-3-1_Tag_occurences.ipynb

@@ -0,0 +1,801 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "71008ae2-4465-45d1-9ad0-6e6d54c99a69",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-12-09T20:15:54.294327Z",
+     "iopub.status.busy": "2025-12-09T20:15:54.294119Z",
+     "iopub.status.idle": "2025-12-09T20:15:54.296418Z",
+     "shell.execute_reply": "2025-12-09T20:15:54.295943Z",
+     "shell.execute_reply.started": "2025-12-09T20:15:54.294313Z"
+    }
+   },
+   "source": [
+    "# Tag occurence percentages"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "1577b529-19b4-471a-af8b-bc331087bb61",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-12-09T20:33:24.090495Z",
+     "iopub.status.busy": "2025-12-09T20:33:24.090266Z",
+     "iopub.status.idle": "2025-12-09T20:33:35.733409Z",
+     "shell.execute_reply": "2025-12-09T20:33:35.732815Z",
+     "shell.execute_reply.started": "2025-12-09T20:33:24.090478Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "==================================================\n",
+      "Tag Analysis for 'anime'\n",
+      "==================================================\n",
+      "Models with tag: 74721\n",
+      "Total models: 232164\n",
+      "Percentage: 32.18%\n",
+      "==================================================\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pathlib import Path\n",
+    "import pandas as pd\n",
+    "import sys\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "\n",
+    "# ============================================\n",
+    "# INPUT: Change these values\n",
+    "# ============================================\n",
+    "csv_file = current_dir.parent / \"data/CSV/models/Civi_models.csv\" # Your CSV file path\n",
+    "tag_to_find = \"anime\"  # Tag to search for\n",
+    "# ============================================\n",
+    "\n",
+    "def calculate_tag_percentage(csv_file, tag_to_find):\n",
+    "    \"\"\"\n",
+    "    Calculate what percentage of models contain a specific tag.\n",
+    "    \"\"\"\n",
+    "    # Read the CSV file\n",
+    "    df = pd.read_csv(csv_file)\n",
+    "    \n",
+    "    # Get all tag columns\n",
+    "    tag_columns = [col for col in df.columns if col.startswith('tag_')]\n",
+    "    \n",
+    "    # Count total models\n",
+    "    total_models = len(df)\n",
+    "    \n",
+    "    # Count models containing the tag (case-insensitive search)\n",
+    "    tag_lower = tag_to_find.lower()\n",
+    "    models_with_tag = 0\n",
+    "    \n",
+    "    for idx, row in df.iterrows():\n",
+    "        # Check if the tag appears in any of the tag columns\n",
+    "        for tag_col in tag_columns:\n",
+    "            tag_value = str(row[tag_col]).lower().strip()\n",
+    "            if tag_value == tag_lower:\n",
+    "                models_with_tag += 1\n",
+    "                break  # Count each model only once\n",
+    "    \n",
+    "    # Calculate percentage\n",
+    "    percentage = (models_with_tag / total_models * 100) if total_models > 0 else 0\n",
+    "    \n",
+    "    return {\n",
+    "        'tag': tag_to_find,\n",
+    "        'count': models_with_tag,\n",
+    "        'total': total_models,\n",
+    "        'percentage': percentage\n",
+    "    }\n",
+    "\n",
+    "# Calculate and display results\n",
+    "result = calculate_tag_percentage(csv_file, tag_to_find)\n",
+    "\n",
+    "print(f\"\\n{'='*50}\")\n",
+    "print(f\"Tag Analysis for '{result['tag']}'\")\n",
+    "print(f\"{'='*50}\")\n",
+    "print(f\"Models with tag: {result['count']}\")\n",
+    "print(f\"Total models: {result['total']}\")\n",
+    "print(f\"Percentage: {result['percentage']:.2f}%\")\n",
+    "print(f\"{'='*50}\\n\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "825dc9b8-ff0e-4afd-b2f1-e4bca1036aea",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-12-09T20:22:12.641296Z",
+     "iopub.status.busy": "2025-12-09T20:22:12.641105Z",
+     "iopub.status.idle": "2025-12-09T20:22:25.293171Z",
+     "shell.execute_reply": "2025-12-09T20:22:25.292530Z",
+     "shell.execute_reply.started": "2025-12-09T20:22:12.641281Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "=== Tag Analysis for '-f' ===\n",
+      "Models with tag: 0\n",
+      "Total models: 232164\n",
+      "Percentage: 0.00%\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pathlib import Path\n",
+    "import pandas as pd\n",
+    "import sys\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "\n",
+    "\n",
+    "def calculate_tag_percentage(csv_file, tag_to_find):\n",
+    "    # Read the CSV file\n",
+    "    df = pd.read_csv(csv_file)\n",
+    "    \n",
+    "    # Get all tag columns\n",
+    "    tag_columns = [col for col in df.columns if col.startswith('tag_')]\n",
+    "    \n",
+    "    # Count total models\n",
+    "    total_models = len(df)\n",
+    "    \n",
+    "    # Count models containing the tag (case-insensitive search)\n",
+    "    tag_lower = tag_to_find.lower()\n",
+    "    models_with_tag = 0\n",
+    "    \n",
+    "    for idx, row in df.iterrows():\n",
+    "        # Check if the tag appears in any of the tag columns\n",
+    "        for tag_col in tag_columns:\n",
+    "            tag_value = str(row[tag_col]).lower().strip()\n",
+    "            if tag_value == tag_lower:\n",
+    "                models_with_tag += 1\n",
+    "                break  # Count each model only once\n",
+    "    \n",
+    "    # Calculate percentage\n",
+    "    percentage = (models_with_tag / total_models * 100) if total_models > 0 else 0\n",
+    "    \n",
+    "    return {\n",
+    "        'tag': tag_to_find,\n",
+    "        'count': models_with_tag,\n",
+    "        'total': total_models,\n",
+    "        'percentage': percentage\n",
+    "    }\n",
+    "\n",
+    "\n",
+    "def analyze_all_tags(csv_file):\n",
+    "\n",
+    "    df = pd.read_csv(csv_file)\n",
+    "    tag_columns = [col for col in df.columns if col.startswith('tag_')]\n",
+    "    total_models = len(df)\n",
+    "    \n",
+    "    # Collect all tags and count occurrences\n",
+    "    tag_counts = {}\n",
+    "    for tag_col in tag_columns:\n",
+    "        for tag in df[tag_col].dropna():\n",
+    "            tag = str(tag).strip()\n",
+    "            if tag:  # Ignore empty strings\n",
+    "                tag_counts[tag] = tag_counts.get(tag, 0) + 1\n",
+    "    \n",
+    "    # Create results DataFrame\n",
+    "    results = []\n",
+    "    for tag, count in tag_counts.items():\n",
+    "        percentage = (count / total_models * 100)\n",
+    "        results.append({\n",
+    "            'tag': tag,\n",
+    "            'count': count,\n",
+    "            'percentage': round(percentage, 2)\n",
+    "        })\n",
+    "    \n",
+    "    results_df = pd.DataFrame(results)\n",
+    "    results_df = results_df.sort_values('count', ascending=False)\n",
+    "    \n",
+    "    return results_df\n",
+    "\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    # Default CSV file path\n",
+    "    csv_file = current_dir.parent / \"data/CSV/models/Civi_models.csv\"\n",
+    "    \n",
+    "    # Check if a specific tag is provided as argument\n",
+    "    if len(sys.argv) > 1:\n",
+    "        tag = sys.argv[1]\n",
+    "        result = calculate_tag_percentage(csv_file, tag)\n",
+    "        \n",
+    "        print(f\"\\n=== Tag Analysis for '{result['tag']}' ===\")\n",
+    "        print(f\"Models with tag: {result['count']}\")\n",
+    "        print(f\"Total models: {result['total']}\")\n",
+    "        print(f\"Percentage: {result['percentage']:.2f}%\\n\")\n",
+    "    else:\n",
+    "        # If no specific tag provided, show all tags\n",
+    "        print(\"\\n=== All Tags Analysis ===\\n\")\n",
+    "        results_df = analyze_all_tags(csv_file)\n",
+    "        print(results_df.to_string(index=False))\n",
+    "        print(f\"\\nTotal unique tags: {len(results_df)}\")\n",
+    "        print(f\"Total models: {len(pd.read_csv(csv_file))}\\n\")\n",
+    "        \n",
+    "        # Show example usage\n",
+    "        print(\"\\nTo search for a specific tag, run:\")\n",
+    "        print(\"  python tag_percentage_calculator.py <tag_name>\")\n",
+    "        print(\"\\nExample:\")\n",
+    "        print(\"  python tag_percentage_calculator.py anime\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "63187f58-9777-4ffb-bdf9-93b191a60241",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-12-09T19:08:49.572641Z",
+     "iopub.status.busy": "2025-12-09T19:08:49.572453Z",
+     "iopub.status.idle": "2025-12-09T19:08:49.634561Z",
+     "shell.execute_reply": "2025-12-09T19:08:49.634109Z",
+     "shell.execute_reply.started": "2025-12-09T19:08:49.572627Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "import json\n",
+    "from collections import defaultdict\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "\n",
+    "\n",
+    "\n",
+    "def load_data(filepath):\n",
+    "    \"\"\"Load the JSON data from file.\"\"\"\n",
+    "    with open(filepath, 'r', encoding='utf-8') as f:\n",
+    "        return json.load(f)\n",
+    "\n",
+    "def calculate_cooccurrence_rate(data, target_tag, cooccurring_tags):\n",
+    "    \"\"\"\n",
+    "    Calculate what percentage of target_tag occurrences co-occur with each tag in cooccurring_tags.\n",
+    "    \n",
+    "    Args:\n",
+    "        data: Dictionary with 'nodes' and 'links'\n",
+    "        target_tag: The main tag to analyze (e.g., \"woman\")\n",
+    "        cooccurring_tags: List of tags to check co-occurrence with (e.g., [\"sexy\", \"pose\"])\n",
+    "    \n",
+    "    Returns:\n",
+    "        Dictionary with results\n",
+    "    \"\"\"\n",
+    "    # Find the target tag's total occurrences\n",
+    "    target_size = None\n",
+    "    for node in data['nodes']:\n",
+    "        if node['id'] == target_tag:\n",
+    "            target_size = node['size']\n",
+    "            break\n",
+    "    \n",
+    "    if target_size is None:\n",
+    "        print(f\"Warning: Tag '{target_tag}' not found in nodes!\")\n",
+    "        return None\n",
+    "    \n",
+    "    print(f\"\\n{'='*60}\")\n",
+    "    print(f\"Analysis for tag: '{target_tag}'\")\n",
+    "    print(f\"{'='*60}\")\n",
+    "    print(f\"Total occurrences of '{target_tag}': {target_size:,}\")\n",
+    "    print()\n",
+    "    \n",
+    "    # Find co-occurrences in links\n",
+    "    results = {}\n",
+    "    for cooccurring_tag in cooccurring_tags:\n",
+    "        cooccurrence_count = 0\n",
+    "        \n",
+    "        # Check both directions in links\n",
+    "        for link in data['links']:\n",
+    "            if (link['source'] == target_tag and link['target'] == cooccurring_tag) or \\\n",
+    "               (link['source'] == cooccurring_tag and link['target'] == target_tag):\n",
+    "                cooccurrence_count = link['value']\n",
+    "                break\n",
+    "        \n",
+    "        if cooccurrence_count > 0:\n",
+    "            percentage = (cooccurrence_count / target_size) * 100\n",
+    "            results[cooccurring_tag] = {\n",
+    "                'count': cooccurrence_count,\n",
+    "                'percentage': percentage\n",
+    "            }\n",
+    "            print(f\"Tag: '{cooccurring_tag}'\")\n",
+    "            print(f\"  Co-occurrences: {cooccurrence_count:,}\")\n",
+    "            print(f\"  Percentage: {percentage:.2f}%\")\n",
+    "            print(f\"  (i.e., {percentage:.2f}% of '{target_tag}' occurrences also have '{cooccurring_tag}')\")\n",
+    "        else:\n",
+    "            results[cooccurring_tag] = {\n",
+    "                'count': 0,\n",
+    "                'percentage': 0.0\n",
+    "            }\n",
+    "            print(f\"Tag: '{cooccurring_tag}'\")\n",
+    "            print(f\"  No co-occurrences found\")\n",
+    "        print()\n",
+    "    \n",
+    "    # Calculate combined co-occurrence (both tags together)\n",
+    "    print(f\"\\n{'='*60}\")\n",
+    "    print(\"Combined Analysis\")\n",
+    "    print(f\"{'='*60}\")\n",
+    "    \n",
+    "    # To find items with ALL tags, we'd need to look at the underlying data\n",
+    "    # With just the graph structure, we can only report individual co-occurrences\n",
+    "    print(f\"Individual co-occurrence rates calculated above.\")\n",
+    "    print(f\"Note: To calculate how often ALL tags appear together,\")\n",
+    "    print(f\"we would need access to the raw item-level data.\")\n",
+    "    \n",
+    "    return results\n",
+    "\n",
+    "def main():\n",
+    "    # Load the data\n",
+    "    filepath = current_dir.parent / \"public/json/nodes_all.json\"\n",
+    "    print(\"Loading data...\")\n",
+    "    data = load_data(filepath)\n",
+    "    print(f\"Loaded {len(data['nodes']):,} nodes and {len(data['links']):,} links\")\n",
+    "    \n",
+    "    # Calculate co-occurrence rates\n",
+    "    target_tag = \"woman\"\n",
+    "    cooccurring_tags = [\"sexy\", \"pose\"]\n",
+    "    \n",
+    "    results = calculate_cooccurrence_rate(data, target_tag, cooccurring_tags)\n",
+    "    \n",
+    "    # Summary\n",
+    "    print(f\"\\n{'='*60}\")\n",
+    "    print(\"SUMMARY\")\n",
+    "    print(f\"{'='*60}\")\n",
+    "    if results:\n",
+    "        for tag, stats in results.items():\n",
+    "            print(f\"'{target_tag}' + '{tag}': {stats['percentage']:.2f}% ({stats['count']:,} occurrences)\")\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    main()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "6e0e8c6b-547e-4899-b001-1d4c6b31476f",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-12-09T19:48:36.009465Z",
+     "iopub.status.busy": "2025-12-09T19:48:36.009249Z",
+     "iopub.status.idle": "2025-12-09T19:48:36.063251Z",
+     "shell.execute_reply": "2025-12-09T19:48:36.062750Z",
+     "shell.execute_reply.started": "2025-12-09T19:48:36.009449Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading data...\n",
+      "Loaded 60,330 nodes and 16,921 links\n",
+      "\n",
+      "================================================================================\n",
+      "Top 100 Co-occurring Tags for: 'anime'\n",
+      "================================================================================\n",
+      "Total occurrences of 'anime': 74,187\n",
+      "\n",
+      "Rank   Tag                            Count        Percentage  \n",
+      "------ ------------------------------ ------------ ------------\n",
+      "1      character                      53,792            72.51%\n",
+      "2      woman                          30,731            41.42%\n",
+      "3      girls                          21,434            28.89%\n",
+      "4      female                         14,286            19.26%\n",
+      "5      style                          11,593            15.63%\n",
+      "6      game character                 9,309             12.55%\n",
+      "7      sexy                           8,876             11.96%\n",
+      "8      male                           4,476              6.03%\n",
+      "9      video game                     3,411              4.60%\n",
+      "10     man                            3,333              4.49%\n",
+      "11     lora                           3,260              4.39%\n",
+      "12     concept                        2,568              3.46%\n",
+      "13     girl                           2,472              3.33%\n",
+      "14     base model                     2,197              2.96%\n",
+      "15     photorealistic                 2,191              2.95%\n",
+      "16     manga                          2,015              2.72%\n",
+      "17     boys                           2,006              2.70%\n",
+      "18     anime character                1,841              2.48%\n",
+      "19     cartoon                        1,759              2.37%\n",
+      "20     game                           1,536              2.07%\n",
+      "21     men                            1,431              1.93%\n",
+      "22     cute                           1,325              1.79%\n",
+      "23     hentai                         1,222              1.65%\n",
+      "24     clothing                       1,131              1.52%\n",
+      "25     furry                          1,114              1.50%\n",
+      "26     realistic                      1,074              1.45%\n",
+      "27     styles                         1,057              1.42%\n",
+      "28     illustration                   971                1.31%\n",
+      "29     characters                     967                1.30%\n",
+      "30     art style                      949                1.28%\n",
+      "31     pokemon                        932                1.26%\n",
+      "32     vtuber                         890                1.20%\n",
+      "33     person                         872                1.18%\n",
+      "34     artstyle                       801                1.08%\n",
+      "35     anime girl                     756                1.02%\n",
+      "36     blue archive                   684                0.92%\n",
+      "37     2d                             666                0.90%\n",
+      "38     fantasy                        643                0.87%\n",
+      "39     art                            633                0.85%\n",
+      "40     poses                          619                0.83%\n",
+      "41     3d                             578                0.78%\n",
+      "42     nsfw                           550                0.74%\n",
+      "43     artist                         544                0.73%\n",
+      "44     genshin impact                 519                0.70%\n",
+      "45     idolmaster                     483                0.65%\n",
+      "46     fire emblem                    481                0.65%\n",
+      "47     fate                           432                0.58%\n",
+      "48     waifu                          406                0.55%\n",
+      "49     azur lane                      399                0.54%\n",
+      "50     dragon ball                    388                0.52%\n",
+      "51     ponyxl                         379                0.51%\n",
+      "52     naruto                         379                0.51%\n",
+      "53     precure                        379                0.51%\n",
+      "54     videogame                      372                0.50%\n",
+      "55     retro                          356                0.48%\n",
+      "56     meme                           355                0.48%\n",
+      "57     arknights                      354                0.48%\n",
+      "58     hololive                       347                0.47%\n",
+      "59     virtual youtuber               347                0.47%\n",
+      "60     umamusume                      333                0.45%\n",
+      "61     falcom                         326                0.44%\n",
+      "62     one piece                      325                0.44%\n",
+      "63     boy                            321                0.43%\n",
+      "64     chibi                          315                0.42%\n",
+      "65     comics                         303                0.41%\n",
+      "66     idolm@ster                     295                0.40%\n",
+      "67     gundam                         294                0.40%\n",
+      "68     bleach                         294                0.40%\n",
+      "69     pose                           288                0.39%\n",
+      "70     guy                            284                0.38%\n",
+      "71     milf                           281                0.38%\n",
+      "72     my hero academia               279                0.38%\n",
+      "73     genshin                        276                0.37%\n",
+      "74     porn                           268                0.36%\n",
+      "75     kawaii                         257                0.35%\n",
+      "76     kantai collection              255                0.34%\n",
+      "77     galgame                        250                0.34%\n",
+      "78     eiyuu densetsu                 246                0.33%\n",
+      "79     animals                        239                0.32%\n",
+      "80     yu-gi-oh!                      236                0.32%\n",
+      "81     comic                          232                0.31%\n",
+      "82     sex                            228                0.31%\n",
+      "83     cinderella girls               227                0.31%\n",
+      "84     kancolle                       223                0.30%\n",
+      "85     huge breasts                   220                0.30%\n",
+      "86     clothes                        219                0.30%\n",
+      "87     digital art                    217                0.29%\n",
+      "88     oc                             216                0.29%\n",
+      "89     scenery                        215                0.29%\n",
+      "90     nintendo                       215                0.29%\n",
+      "91     manhwa                         214                0.29%\n",
+      "92     final fantasy                  211                0.28%\n",
+      "93     nikke                          211                0.28%\n",
+      "94     cosplay                        208                0.28%\n",
+      "95     beautiful                      207                0.28%\n",
+      "96     dragon ball z                  207                0.28%\n",
+      "97     concepts                       207                0.28%\n",
+      "98     videogame character            207                0.28%\n",
+      "99     thick thighs                   205                0.28%\n",
+      "100    wide hips                      202                0.27%\n",
+      "\n",
+      "================================================================================\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pathlib import Path\n",
+    "import json\n",
+    "from collections import defaultdict\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "\n",
+    "def load_data(filepath):\n",
+    "    \"\"\"Load the JSON data from file.\"\"\"\n",
+    "    with open(filepath, 'r', encoding='utf-8') as f:\n",
+    "        return json.load(f)\n",
+    "\n",
+    "def get_top_cooccurrences(data, target_tag, top_n=10):\n",
+    "    \"\"\"\n",
+    "    Find the top N tags that co-occur with the target tag.\n",
+    "    \n",
+    "    Args:\n",
+    "        data: Dictionary with 'nodes' and 'links'\n",
+    "        target_tag: The main tag to analyze (e.g., \"woman\")\n",
+    "        top_n: Number of top co-occurring tags to return (default: 10)\n",
+    "    \n",
+    "    Returns:\n",
+    "        List of tuples (tag, count, percentage) sorted by count\n",
+    "    \"\"\"\n",
+    "    # Find the target tag's total occurrences\n",
+    "    target_size = None\n",
+    "    for node in data['nodes']:\n",
+    "        if node['id'] == target_tag:\n",
+    "            target_size = node['size']\n",
+    "            break\n",
+    "    \n",
+    "    if target_size is None:\n",
+    "        print(f\"Error: Tag '{target_tag}' not found in nodes!\")\n",
+    "        return None, None\n",
+    "    \n",
+    "    # Find all co-occurrences in links\n",
+    "    cooccurrences = []\n",
+    "    \n",
+    "    for link in data['links']:\n",
+    "        if link['source'] == target_tag:\n",
+    "            cooccurrences.append({\n",
+    "                'tag': link['target'],\n",
+    "                'count': link['value']\n",
+    "            })\n",
+    "        elif link['target'] == target_tag:\n",
+    "            cooccurrences.append({\n",
+    "                'tag': link['source'],\n",
+    "                'count': link['value']\n",
+    "            })\n",
+    "    \n",
+    "    # Sort by count (descending) and take top N\n",
+    "    cooccurrences.sort(key=lambda x: x['count'], reverse=True)\n",
+    "    top_cooccurrences = cooccurrences[:top_n]\n",
+    "    \n",
+    "    # Calculate percentages\n",
+    "    results = []\n",
+    "    for item in top_cooccurrences:\n",
+    "        percentage = (item['count'] / target_size) * 100\n",
+    "        results.append((item['tag'], item['count'], percentage))\n",
+    "    \n",
+    "    return results, target_size\n",
+    "\n",
+    "def display_results(target_tag, results, target_size, top_n):\n",
+    "    \"\"\"Display the results in a formatted table.\"\"\"\n",
+    "    if results is None:\n",
+    "        return\n",
+    "    \n",
+    "    print(f\"\\n{'='*80}\")\n",
+    "    print(f\"Top {top_n} Co-occurring Tags for: '{target_tag}'\")\n",
+    "    print(f\"{'='*80}\")\n",
+    "    print(f\"Total occurrences of '{target_tag}': {target_size:,}\\n\")\n",
+    "    \n",
+    "    if not results:\n",
+    "        print(f\"No co-occurrences found for '{target_tag}'\")\n",
+    "        return\n",
+    "    \n",
+    "    # Print header\n",
+    "    print(f\"{'Rank':<6} {'Tag':<30} {'Count':<12} {'Percentage':<12}\")\n",
+    "    print(f\"{'-'*6} {'-'*30} {'-'*12} {'-'*12}\")\n",
+    "    \n",
+    "    # Print results\n",
+    "    for i, (tag, count, percentage) in enumerate(results, 1):\n",
+    "        print(f\"{i:<6} {tag:<30} {count:<12,} {percentage:>10.2f}%\")\n",
+    "    \n",
+    "    print(f\"\\n{'='*80}\\n\")\n",
+    "\n",
+    "def main():\n",
+    "    # Load the data\n",
+    "    filepath = current_dir.parent / \"public/json/nodes_all.json\"\n",
+    "    print(\"Loading data...\")\n",
+    "    data = load_data(filepath)\n",
+    "    print(f\"Loaded {len(data['nodes']):,} nodes and {len(data['links']):,} links\")\n",
+    "    \n",
+    "    # Analyze different tags\n",
+    "    target_tags = [\"anime\"]  # Add more tags here to analyze multiple\n",
+    "    top_n = 100\n",
+    "    \n",
+    "    for target_tag in target_tags:\n",
+    "        results, target_size = get_top_cooccurrences(data, target_tag, top_n)\n",
+    "        display_results(target_tag, results, target_size, top_n)\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    main()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "e8af35af-a4b9-4011-b8c3-5ec8e75ce6c1",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-12-09T19:47:44.592679Z",
+     "iopub.status.busy": "2025-12-09T19:47:44.592464Z",
+     "iopub.status.idle": "2025-12-09T19:47:44.649077Z",
+     "shell.execute_reply": "2025-12-09T19:47:44.648523Z",
+     "shell.execute_reply.started": "2025-12-09T19:47:44.592664Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading data...\n",
+      "Loaded 60,330 nodes and 16,921 links\n",
+      "\n",
+      "\n",
+      "================================================================================\n",
+      "Co-occurrence Analysis: 'anime' + 'dragon ball'\n",
+      "================================================================================\n",
+      "\n",
+      "Total occurrences of 'anime': 74,187\n",
+      "Total occurrences of 'dragon ball': 479\n",
+      "\n",
+      "Items with BOTH tags: 388\n",
+      "\n",
+      ">>> 0.52% of 'anime' occurrences also have 'dragon ball'\n",
+      "\n",
+      "================================================================================\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pathlib import Path\n",
+    "import json\n",
+    "from collections import defaultdict\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "\n",
+    "def load_data(filepath):\n",
+    "    \"\"\"Load the JSON data from file.\"\"\"\n",
+    "    with open(filepath, 'r', encoding='utf-8') as f:\n",
+    "        return json.load(f)\n",
+    "\n",
+    "def get_tag_cooccurrence(data, tag1, tag2):\n",
+    "    \"\"\"\n",
+    "    Find what percentage of tag1 occurrences also have tag2.\n",
+    "    \n",
+    "    Args:\n",
+    "        data: Dictionary with 'nodes' and 'links'\n",
+    "        tag1: Primary tag to analyze (e.g., \"cat\")\n",
+    "        tag2: Secondary tag to check for (e.g., \"dog\")\n",
+    "    \n",
+    "    Returns:\n",
+    "        Dictionary with co-occurrence information\n",
+    "    \"\"\"\n",
+    "    # Find the tags' total occurrences\n",
+    "    tag1_size = None\n",
+    "    tag2_size = None\n",
+    "    \n",
+    "    for node in data['nodes']:\n",
+    "        if node['id'] == tag1:\n",
+    "            tag1_size = node['size']\n",
+    "        if node['id'] == tag2:\n",
+    "            tag2_size = node['size']\n",
+    "    \n",
+    "    if tag1_size is None:\n",
+    "        print(f\"Error: Tag '{tag1}' not found in nodes!\")\n",
+    "        return None\n",
+    "    \n",
+    "    if tag2_size is None:\n",
+    "        print(f\"Error: Tag '{tag2}' not found in nodes!\")\n",
+    "        return None\n",
+    "    \n",
+    "    # Find co-occurrence count in links\n",
+    "    # This represents how many items have BOTH tag1 AND tag2\n",
+    "    cooccurrence_count = 0\n",
+    "    \n",
+    "    for link in data['links']:\n",
+    "        if (link['source'] == tag1 and link['target'] == tag2) or \\\n",
+    "           (link['source'] == tag2 and link['target'] == tag1):\n",
+    "            cooccurrence_count = link['value']\n",
+    "            break\n",
+    "    \n",
+    "    # Calculate percentage: what % of tag1 items also have tag2\n",
+    "    percentage_with_tag2 = (cooccurrence_count / tag1_size) * 100 if tag1_size > 0 else 0\n",
+    "    \n",
+    "    return {\n",
+    "        'primary_tag': tag1,\n",
+    "        'secondary_tag': tag2,\n",
+    "        'primary_tag_total': tag1_size,\n",
+    "        'secondary_tag_total': tag2_size,\n",
+    "        'cooccurrence_count': cooccurrence_count,\n",
+    "        'percentage_with_secondary': percentage_with_tag2\n",
+    "    }\n",
+    "\n",
+    "def display_cooccurrence_results(result):\n",
+    "    \"\"\"Display the co-occurrence results in a formatted way.\"\"\"\n",
+    "    if result is None:\n",
+    "        return\n",
+    "    \n",
+    "    print(f\"\\n{'='*80}\")\n",
+    "    print(f\"Co-occurrence Analysis: '{result['primary_tag']}' + '{result['secondary_tag']}'\")\n",
+    "    print(f\"{'='*80}\\n\")\n",
+    "    \n",
+    "    print(f\"Total occurrences of '{result['primary_tag']}': {result['primary_tag_total']:,}\")\n",
+    "    print(f\"Total occurrences of '{result['secondary_tag']}': {result['secondary_tag_total']:,}\")\n",
+    "    print(f\"\\nItems with BOTH tags: {result['cooccurrence_count']:,}\")\n",
+    "    print(f\"\\n>>> {result['percentage_with_secondary']:.2f}% of '{result['primary_tag']}' occurrences also have '{result['secondary_tag']}'\")\n",
+    "    \n",
+    "    print(f\"\\n{'='*80}\\n\")\n",
+    "\n",
+    "def analyze_multiple_pairs(data, tag_pairs):\n",
+    "    \"\"\"\n",
+    "    Analyze multiple tag pairs at once.\n",
+    "    \n",
+    "    Args:\n",
+    "        data: Dictionary with 'nodes' and 'links'\n",
+    "        tag_pairs: List of tuples, each containing two tags to compare\n",
+    "    \"\"\"\n",
+    "    results = []\n",
+    "    \n",
+    "    for tag1, tag2 in tag_pairs:\n",
+    "        result = get_tag_cooccurrence(data, tag1, tag2)\n",
+    "        if result:\n",
+    "            results.append(result)\n",
+    "            display_cooccurrence_results(result)\n",
+    "    \n",
+    "    return results\n",
+    "\n",
+    "def main():\n",
+    "    # Load the data\n",
+    "    filepath = current_dir.parent / \"public/json/nodes_all.json\"\n",
+    "    print(\"Loading data...\")\n",
+    "    data = load_data(filepath)\n",
+    "    print(f\"Loaded {len(data['nodes']):,} nodes and {len(data['links']):,} links\\n\")\n",
+    "    \n",
+    "    # Analyze: What percentage of \"cat\" occurrences also have \"dog\"?\n",
+    "    primary_tag = \"anime\"      # The main tag you're interested in\n",
+    "    secondary_tag = \"dragon ball\"    # The tag you want to check for\n",
+    "    \n",
+    "    result = get_tag_cooccurrence(data, primary_tag, secondary_tag)\n",
+    "    display_cooccurrence_results(result)\n",
+    "    \n",
+    "    # You can also check the reverse: What percentage of \"dog\" occurrences also have \"cat\"?\n",
+    "    # result_reverse = get_tag_cooccurrence(data, \"dog\", \"cat\")\n",
+    "    # display_cooccurrence_results(result_reverse)\n",
+    "    \n",
+    "    # Option: Analyze multiple pairs at once\n",
+    "    # Uncomment the lines below to analyze multiple pairs\n",
+    "    \"\"\"\n",
+    "    tag_pairs = [\n",
+    "        (\"cat\", \"dog\"),\n",
+    "        (\"boy\", \"anime\"),\n",
+    "        (\"girl\", \"anime\"),\n",
+    "        (\"man\", \"photorealistic\")\n",
+    "    ]\n",
+    "    results = analyze_multiple_pairs(data, tag_pairs)\n",
+    "    \"\"\"\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    main()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a79ee96c-a060-4cc5-82d8-5642ccbef328",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

jupyter_notebooks/Section_2-3-4_Figure_8_Step_1_LLM_annotation.ipynb

@@ -466,7 +466,7 @@
     "            'max_new_tokens': 512,\n",
     "            'temperature': 0.05,\n",
     "            'do_sample': True,\n",
-    "            'top_p': 0.8,\n",
+    "            'top_p': 1.0,\n",
     "        }\n",
     "    },\n",
     "    'gemma': {\n",
@@ -477,7 +477,7 @@
     "            'max_new_tokens': 512,\n",
     "            'temperature': 0.1,\n",
     "            'do_sample': True,\n",
-    "            'top_p': 0.9,\n",
+    "            'top_p': 1.0,\n",
     "        }\n",
     "    },\n",
     "    'qwen': {\n",
@@ -489,7 +489,7 @@
     "            llm_int8_has_fp16_weight=False\n",
     "        ),\n",
     "        'generation_params': {\n",
-    "            'max_new_tokens': 100,\n",
+    "            'max_new_tokens': 512,\n",
     "            'temperature': 0.1,\n",
     "            'do_sample': False,\n",
     "        }\n",
@@ -1428,15 +1428,12 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "a55a5e30-83f3-4f7c-a537-b1216d4e8a07",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-12-08T23:57:35.685431Z",
-     "iopub.status.busy": "2025-12-08T23:57:35.685314Z",
-     "iopub.status.idle": "2025-12-08T23:59:48.656498Z",
-     "shell.execute_reply": "2025-12-08T23:59:48.655927Z",
-     "shell.execute_reply.started": "2025-12-08T23:57:35.685419Z"
+     "iopub.execute_input": "2025-12-09T22:16:21.002786Z",
+     "iopub.status.busy": "2025-12-09T22:16:21.002337Z"
     }
    },
    "outputs": [
@@ -1458,18 +1455,71 @@
       "\n",
       "Device: cuda\n",
       "Loading tokenizer...\n",
-      "✅ Tokenizer loaded\n"
+      "✅ Tokenizer loaded\n",
+      "Loading model (this may take several minutes)...\n"
      ]
     },
     {
-     "ename": "NameError",
-     "evalue": "name 'BitsAndBytesConfig' is not defined",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
-      "\u001b[31mNameError\u001b[39m                                 Traceback (most recent call last)",
-      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 82\u001b[39m\n\u001b[32m     78\u001b[39m     tokenizer.pad_token = tokenizer.eos_token\n\u001b[32m     80\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33m✅ Tokenizer loaded\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m---> \u001b[39m\u001b[32m82\u001b[39m quantization_config = \u001b[43mBitsAndBytesConfig\u001b[49m(\n\u001b[32m     83\u001b[39m     load_in_8bit=\u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[32m     84\u001b[39m )\n\u001b[32m     87\u001b[39m \u001b[38;5;66;03m# Load model with optimizations\u001b[39;00m\n\u001b[32m     88\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33mLoading model (this may take several minutes)...\u001b[39m\u001b[33m\"\u001b[39m)\n",
-      "\u001b[31mNameError\u001b[39m: name 'BitsAndBytesConfig' is not defined"
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "`torch_dtype` is deprecated! Use `dtype` instead!\n",
+      "Loading checkpoint shards: 100%|██████████| 9/9 [02:42<00:00, 18.06s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Model loaded\n",
+      "VRAM used: 21.40 GB\n",
+      "\n",
+      "Loading raw input CSV...\n",
+      "Loaded 50861 rows from input file\n",
+      "Found existing annotations, merging...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/tmp/ipykernel_3104208/1997558719.py:113: DtypeWarning: Columns (52,53,54,55,56) have mixed types. Specify dtype option on import or set low_memory=False.\n",
+      "  existing_df = pd.read_csv(output_file)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Existing annotations has 50861 rows\n",
+      "Merged annotations, continuing with 50861 total rows\n",
+      "✅ Loaded professions.csv\n",
+      "✅ Loaded profession mapping with 9 categories\n",
+      "Loaded 50861 rows\n",
+      "\n",
+      "Profession categories (9):\n",
+      "  - actor\n",
+      "  - adult performer\n",
+      "  - singer/musician\n",
+      "  - model\n",
+      "  - online personality\n",
+      "  - public figure\n",
+      "  - voice actor/ASMR\n",
+      "  - sports professional\n",
+      "  - tv personality\n",
+      "\n",
+      "Creating prompts...\n",
+      "✅ Prompts created\n",
+      "Resuming from index 8810\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 0/42051 [00:00<?, ?it/s]/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/bitsandbytes/autograd/_functions.py:181: UserWarning: MatMul8bitLt: inputs will be cast from torch.bfloat16 to float16 during quantization\n",
+      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n",
+      "Mistral Local:   0%|          | 7/42051 [00:57<93:01:03,  7.96s/it] "
      ]
     }
    ],
@@ -1481,7 +1531,7 @@
     "from pathlib import Path\n",
     "from tqdm import tqdm\n",
     "import torch\n",
-    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
     "\n",
     "current_dir = Path.cwd()\n",
     "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",

misc/query_indicies/mistral24_local_query_index.txt

@@ -1 +1 @@
-3890
+14840