WIP: Adding AI #6

Draft
unfunny wants to merge 2 commits from chromeai into master
26 changed files with 12633 additions and 2 deletions

33
chromeAICheck.html Normal file
View File

@@ -0,0 +1,33 @@
<!DOCTYPE html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<!-- /\ you see this? this is the sum of those mobile optimizations. one line. yeah. you can leave now.-->
<meta property="og:title" content="Doofensmirtz ONLINE" />
<meta property="og:type" content="website" />
<meta property="og:url" content="https://doofensmirtzevil.neocities.org/" />
<meta property="og:image" content="https://doofensmirtzevil.neocities.org/newlogo.png" />
<title>DOOFENSMIRTZ ONLINE</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Libre+Barcode+39+Extended+Text&display=swap" rel="stylesheet">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Sono&display=swap" rel="stylesheet">
<link href="/normalbordem.css" rel="stylesheet" type="text/css" media="all">
<script src="/src/chromeaichk.js"></script>
<!--<script src="cdnewyear.js"></script> \
Fun Fact: It is not currently new years-->
</head>
<body>
<h1>Chrome AI Model check </h1>
<p>some features of this site use on-device AI to power assitant features and the upcoming infinite ikea.</p>
<br><img src="anibar.gif"><br>
<div id="throbber">Please Wait... Checking your device.</div>
<br><img src="anibar.gif"><br>
<a href=javascript:history.back()><img src="/eternalbliss/BLUEamnesiac'sWTV.gif" title="click me to go back"></img></a>
</body>
<script> aichk();</script>

View File

@@ -96,5 +96,6 @@ Current Viewers:<script language="javascript" type="text/javascript" src="//coun
<img src=anibar.gif>
<br>
<h6 style="color: grey">psst. if you get tired of all the flashy stuff, <a href=index2.html style="color: grey;">check out the lite version of this site</a>.</h6>
<h6 style="color: grey">Utilities<br><a href="chromeAICheck.html">ai check</a></h6>
<!--<h6 style="color: grey">brb, lite mode site is broken rn</h6>-->
</body>

6646
infikea-brainstorm Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -4,8 +4,10 @@ import http.server
import socketserver
import subprocess
import sys
import argparse
PORT = 8080
BIND_ADDRESS = "localhost" # Default to localhost for security
class Handler(http.server.SimpleHTTPRequestHandler):
def log_message(self, format, *args):
@@ -17,7 +19,21 @@ class Handler(http.server.SimpleHTTPRequestHandler):
super().end_headers()
if __name__ == "__main__":
with socketserver.TCPServer(("", PORT), Handler) as httpd:
print(f"Server running at http://localhost:{PORT}")
parser = argparse.ArgumentParser(description="Simple HTTP server for testing")
parser.add_argument("-b", "--bind", default="localhost",
help="Bind address (default: localhost). Use '0.0.0.0' to bind all interfaces")
parser.add_argument("-p", "--port", type=int, default=PORT,
help=f"Port number (default: {PORT})")
args = parser.parse_args()
# Convert "localhost" to "" for bind address, or use provided address
bind_addr = "" if args.bind == "0.0.0.0" else args.bind
print(f"Binding to: {args.bind}")
print(f"Port: {args.port}")
with socketserver.TCPServer((bind_addr, args.port), Handler) as httpd:
url = f"http://{'localhost' if args.bind == 'localhost' else args.bind}:{args.port}"
print(f"Server running at {url}")
sys.stdout.flush()
httpd.serve_forever()

32
src/chromeaichk.js Normal file
View File

@@ -0,0 +1,32 @@
// @ts-check
// Uh yeah this code is terrible, i DO NOT care
// womp womp
//also womp womp to the anti-ai people, wah the water wah it's on your localhost i do not care, your PC is not magically siphoning water from the nearest river
var nochromeai = false
async function aichk() {
// First check for ChromeAI
try {
const availability = await Summarizer.availability({outputLanguage: 'en'});
} catch (e) {
console.error("ChromeAI functions do not exist. Moving on, but I think you should know: ", e);
var nochromeai = true
}
if (nochromeai == true) {
document.getElementById("throbber").innerHTML = "Chrome AI API call failed. Check console for more details. <br> <h6>sucks to suck</h6>";
} else {
document.getElementById("throbber").innerHTML = "Chrome AI API exists!";
}
// Check Randomhack LLM reachability
// Last resort, cloud-based (as in, hosted by myself in a homelab
// a 401 is expected here.
await fetch("https://llmapi.randomhack.com/v1/models")
.then(r => { console.log("[LLM Ping] v1 reachable, status:", r.status); document.getElementById("throbber").innerHTML += "<br>Connection to Randomhack LLM service succeeded!"; })
.catch(e => { console.error("[LLM Ping] v1 unreachable:", e.message); document.getElementById("throbber").innerHTML += "<br>Connection to Randomhack LLM service failed.<br> AI Functionality will not be enabled.<br> <h6> looser</h6>"; });
if (nochromeai == true ) {
document.getElementById("throbber").innerHTML += "<br> <strong>AI will not be running on-device.</strong>";
} else {
document.getElementById("throbber").innerHTML += "<strong>аll systems go! <br> AI is fully functional, you are ready to explore doofonline to it's fullest</strong>";
}
}

View File

@@ -0,0 +1,615 @@
# Horror Prompt Optimization System - Documentation
## 1. Project Overview and Purpose
The **Horror Prompt Optimization System** is an automated workflow designed to refine prompts for generating horror text adventure maze content through iterative AI-assisted optimization. The system uses a teacher-student learning paradigm where a powerful "teacher" model (Qwen-3.5-9B) optimizes prompts for smaller models based on known quirks and human feedback.
### Key Objectives
- Generate high-quality, original horror text adventure content
- Optimize prompts to minimize repetition while maintaining narrative coherence
- Provide multi-dimensional grading of generated rooms (horror quality, creativity, flow, etc.)
- Use teacher-student learning with model-specific quirks for targeted optimization
- Enable human-in-the-loop feedback integration
---
## 2. Architecture Diagram (Text-Based)
```
┌─────────────────────────────────────────────────────────────────────┐
│ MAIN WORKFLOW CONTROLLER │
│ (main.py) │
└─────────────────────────────────────────────────────────────────────┘
┌───────────────────────────┼───────────────────────────┐
▼ ▼ ▼
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
│ PHASE 1A │ │ PHASE 1B/1C │ │ PHASE 2A │
│ Golden Prompt │ │ Teacher Opt │ │ Human Grading │
│ │ │ (Coarse/Fine) │ │ (8 Rooms) │
└─────────────────┘ └─────────────────┘ └─────────────────┘
│ │ │
▼ ▼ ▼
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
│ PHASE 2B │ │ QUIRKS DB │ │ STORAGE SYSTEM │
│ Teacher Iter. │ │ (Model Quirks) │ │ (JSON Files) │
└─────────────────┘ └─────────────────┘ └─────────────────┘
┌─────────────────────────────────────────────────────────────────────┐
│ COMPONENTS │
├─────────────────────────────────────────────────────────────────────┤
│ │
│ ┌──────────────────┐ ┌──────────────────┐ │
│ │ API CLIENT │ │ GRADING │ │
│ │ (llm/api_client) │ │ ENGINE │ │
│ └──────────────────┘ └──────────────────┘ │
│ │ │ │
│ ▼ ▼ │
│ ┌──────────────────┐ ┌──────────────────┐ │
│ │ TEACHER │ │ STORAGE │ │
│ │ OPTIMIZER │ │ (data/storage) │ │
│ └──────────────────┘ └──────────────────┘ │
│ │
└─────────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────────┐
│ DATA FLOW │
├─────────────────────────────────────────────────────────────────────┤
│ │
│ Human Prompt ──► Phase 1A ──► Teacher Opt (Phase 1B/1C) ────► Optimized Prompts
│ │
│ Optimized Prompt ──► Generate Rooms (8 per iteration) ──────► Grading Engine
│ │
│ Human Feedback ◄──────────────────────────────────────────────┘
│ │ ▲
│ ▼ │
│ Teacher Iteration (Phase 2B) ◄────────────────────────┘
│ │
└─────────────────────────────────────────────────────────────────────┘
```
---
## 3. Directory Structure with Explanations
```
ikeatraining/
├── main.py # Main workflow controller orchestrates all phases
├── config/
│ ├── llm_api_config.py # LLM API configuration (OpenAI/LM Studio/LiteLLM)
│ │ - Model definitions with temperature/max_tokens
│ │ - API base URL and authentication
│ │ - Grading configuration thresholds
│ │ - Phase iteration counts
│ │
├── llm/
│ ├── api_client.py # Generic LLM API client for OpenAI-compatible APIs
│ │ - Handles chat/completions requests
│ │ - Supports LM Studio, LiteLLM proxies
│ │
│ ├── teacher_optimizer.py # Teacher model optimizer using quirks database
│ │ - Optimizes prompts based on known model quirks
│ │ - Integrates human feedback for iterative improvement
│ │
│ └── grading_engine.py # Multi-dimensional room grading engine
│ - Grades horror quality, coherence, repetition, creativity, flow
│ - Batch processing of multiple rooms
├── data/
│ └── storage.py # JSON file I/O utilities for all data persistence
│ - Prompts directory management
│ - Feedback storage per model
│ - Quirks database loading/saving
├── quirks/ # Model-specific quirks database (JSON files)
│ └── model_quirks.json # Sample quirks for all supported models
│ - Defines known weaknesses/biases per model
├── prompts/ # Optimized prompt storage directory
│ ├── model_qwen_3.5_1b_final_prompt.txt # Final optimized prompts per model
│ └── ... # One file per model
├── feedback/ # Human grading feedback storage
│ └── human_feedback_<model>.json # Grading results per room iteration
└── data/storage.json # Centralized state storage (optional)
Directory Structure:
- config/: Configuration files for API, models, and workflow parameters
- llm/: Core AI components (API client, optimizer, grader)
- quirks/: Model-specific behavioral characteristics database
- prompts/: Final optimized prompts after all iterations complete
- feedback/: Human evaluation results from Phase 2A grading sessions
```
---
## 4. Configuration Details
### API Configuration (`config/llm_api_config.py`)
**OpenAI-Compatible API Setup:**
```python
OPENAI_API_BASE = "http://localhost:1234/v1" # LM Studio default port
OPENAI_API_KEY = "your-api-key-here" # Replace with actual key or use LiteLLM proxy
```
**Supported Models:**
| Model Name | Temperature | Max Tokens | Role |
|------------|-------------|------------|------|
| qwen-3.5-0.5b | 0.7 | 2048 | Student model |
| qwen-3.5-1b | 0.7 | 2048 | Student model |
| gemma-3-2b | 0.7 | 2048 | Student model |
| phi-3-mini-3.8b | 0.7 | 2048 | Student model |
| qwen-3.5-9b | 0.7 | 4096 | **Teacher Model** (optimization) |
### Grading Thresholds (`config/grading_config.py`)
```python
GRADING_THRESHOLDS = {
"horror_quality": {"min_pass": 3, "max_fail": 1}, # Scale: 1-5
"coherence_score": {"min_pass": 0.7, "max_fail": 0.4}, # Scale: 0-1
"repetition_rate": {"min_pass": 0.2, "max_fail": 0.6}, # Scale: 0-1 (lower better)
"creativity_index": {"min_pass": 3, "max_fail": 1}, # Scale: 1-5
"narrative_flow": {"min_pass": 3, "max_fail": 1}, # Scale: 1-5
}
```
**Grading Dimensions:**
| Dimension | Scale | Pass Threshold | Description |
|-----------|-------|----------------|-------------|
| Horror Quality | 1-5 | ≥3 | Atmosphere, tension, horror elements effectiveness |
| Coherence Score | 0-1 | ≥0.7 | Narrative consistency and logical flow |
| Repetition Rate | 0-1 | ≤0.2 | Word/phrasing duplication (lower is better) |
| Creativity Index | 1-5 | ≥3 | Originality of horror elements |
| Narrative Flow | 1-5 | ≥3 | Pacing and transitions between rooms |
### Phase Configuration (`config/llm_api_config.py`)
```python
PHASE_CONFIG = {
"phase_1a_golden_prompt_iterations": 1, # Human writes once initially
"phase_1b_teacher_optimization_iterations": 3,
"phase_1c_fine_tuning_iterations": 5, # Multiple iterations per model
"phase_2a_human_grading_rooms": 8, # Rooms tested per iteration
"phase_2b_teacher_iteration_iterations": 4, # Teacher refinement rounds
}
GRADING_CONFIG = {
"rooms_per_iteration": 8, # Test rooms generated per prompt
"min_rooms_for_grading": 5, # Minimum rooms for valid grading
}
```
---
## 5. Workflow Phases Explained in Detail
### Phase 1A: Human Writes Golden Prompt
**Purpose:** Establish the baseline prompt that will be optimized throughout the workflow.
**Process:**
1. User manually writes a comprehensive horror text adventure prompt into `prompts/golden_prompt.txt`
2. The system reads this golden prompt and uses it as the starting point for all optimizations
3. This phase runs once at the beginning of each optimization cycle
**Example Golden Prompt Content:**
```
You are a horror text adventure generator. Create immersive and terrifying maze rooms
with strong atmosphere, tension-building elements, and original horror concepts.
Each room should connect logically to previous rooms while introducing fresh scares.
Focus on sensory details, pacing, and narrative coherence. Avoid repetition in phrasing
while maintaining consistent tone and style throughout the experience.
```
### Phase 1B: Teacher Coarse Optimization (Using Quirks)
**Purpose:** Apply known model quirks to make initial prompt adjustments for each student model.
**Process:**
1. Load quirks database for target model from `quirks/model_<name>_quirks.json`
2. Teacher optimizer (Qwen-3.5-9B) analyzes golden prompt with quirk context
3. Generate coarse optimization suggestions based on known weaknesses
4. Save initial optimized prompt to prompts directory
**Model Quirks Example:**
```json
{
"qwen-3.5-1b": {
"hallucinates_details": false,
"overuses_adjectives": true,
"short_responses": false,
"struggles_with_pacing": false
}
}
```
### Phase 1C: Fine Tuning (Multiple Iterations)
**Purpose:** Refine prompts through iterative teacher optimization without human feedback.
**Process:**
1. For each student model, run multiple fine-tuning iterations
2. Teacher optimizer applies quirks-based adjustments repeatedly
3. Each iteration builds on previous optimizations
4. Track improvements in prompt quality over iterations
### Phase 2A: Human Grading (Test Rooms)
**Purpose:** Evaluate generated horror content through human expert review.
**Process:**
1. Generate 8 test rooms using the current optimized prompt for a model
2. Human grader rates each room on 5 dimensions:
- Horror Quality (1-5)
- Coherence Score (0-1)
- Repetition Rate (0-1, lower better)
- Creativity Index (1-5)
- Narrative Flow (1-5)
3. Save grading results to `feedback/human_feedback_<model>.json`
**Grading Output Format:**
```json
{
"room_content": "...",
"grades": {
"horror_quality": 4,
"coherence_score": 0.85,
"repetition_rate": 0.3,
"creativity_index": 4,
"narrative_flow": 4
},
"analysis": "Room had strong atmosphere but some repetitive phrasing..."
}
```
### Phase 2B: Teacher Iteration (Using Human Feedback)
**Purpose:** Use human grading feedback to further refine prompts.
**Process:**
1. Load quirks database and human feedback from Phase 2A
2. Teacher optimizer analyzes both data sources together
3. Generate refined prompt adjustments addressing identified issues
4. Repeat for specified number of iterations (typically 4)
5. Final optimized prompts saved to prompts directory
---
## 6. How to Run the System Step-by-Step
### Prerequisites
1. **LM Studio or LiteLLM Setup:**
- Start LM Studio server on port 1234, OR
- Configure LiteLLM proxy pointing to your local LLM server
2. **API Key Configuration:**
```python
# In config/llm_api_config.py
OPENAI_API_KEY = "your-api-key-here"
```
### Step 1: Create Golden Prompt File
```bash
# Navigate to project directory
cd S:\doofonline\utils\ikeatraining
# Create golden prompt file
echo "Your horror text adventure prompt here..." > prompts/golden_prompt.txt
```
### Step 2: Run the Workflow
```bash
# Execute main.py - selects model, runs all phases interactively
python main.py
```
**Interactive Flow:**
1. System displays available models with API availability status
2. User selects a model to test (or accepts default)
3. System automatically executes all 5 workflow phases:
- Phase 1A: Reads golden prompt
- Phase 1B: Coarse optimization using quirks
- Phase 1C: Fine-tuning iterations
- Phase 2A: Human grading of test rooms
- Phase 2B: Teacher iteration with feedback
### Step 3: Review Results
```bash
# Check optimized prompts
cat prompts/model_qwen_3.5_1b_final_prompt.txt
# View human grading feedback
cat feedback/human_feedback_model_qwen_3.5_1b.json
# View quirks database
cat quirks/model_quirks.json
```
---
## 7. Model Selection Process
### Automatic API Validation
The system automatically validates model selection against available models on the configured API:
**Validation Flow:**
1. Fetch `/models` endpoint from API using provided credentials
2. Extract list of available model IDs
3. Cross-reference with configured `MODELS` dictionary
4. Display only valid models to user (marked with [OK] or [?])
5. User selects from validated options
**Example Output:**
```
Found 5 models on API
Available models:
1. qwen-3.5-0.5b ([OK]) temp=0.7, max_tokens=2048
2. qwen-3.5-1b ([OK]) temp=0.7, max_tokens=2048
3. gemma-3-2b ([?]) temp=0.7, max_tokens=2048
```
### Manual Model Configuration
If API validation fails or you want to test models not on the server:
**Edit `config/llm_api_config.py`:**
```python
MODELS = {
"qwen-3.5-0.5b": {"temperature": 0.7, "max_tokens": 2048},
# Add or modify models as needed
}
API_CONFIG = {
"model_name": "qwen-3.5-1b", # Default test model
"api_base": OPENAI_API_BASE,
"api_key": OPENAI_API_KEY,
}
```
---
## 8. API Setup Instructions for LM Studio/LiteLLM
### Option A: LM Studio (Recommended)
**Step 1: Start LM Studio Server**
1. Launch LM Studio desktop application
2. Click the server icon (top-right corner) to start local server
3. Default port: `1234`
4. Ensure your model is loaded and ready
**Step 2: Configure API Key**
- LM Studio doesn't require an API key for local usage
- Leave `OPENAI_API_KEY` empty or set to `"sk-"` prefix if needed
**Step 3: Verify Connection**
```python
# Test connection in Python
from llm.api_client import get_client
client = get_client()
response = client.chat_completion([{"role": "user", "content": "Hello"}])
print(response["content"])
```
### Option B: LiteLLM Proxy
**Step 1: Install LiteLLM**
```bash
pip install litellm
```
**Step 2: Start LiteLLM Server**
```bash
# Point to your local LLM server (e.g., LM Studio, Ollama)
litellm --api-base http://localhost:11434/v1
```
**Step 3: Configure API Key**
- Use any valid key or leave empty for LiteLLM proxy mode
- Set `OPENAI_API_KEY = "your-key"` in config file
### Option C: Ollama (Alternative)
**Step 1: Pull Model**
```bash
ollama pull qwen2.5:7b
```
**Step 2: Start Ollama Server**
- Ollama runs automatically when you pull a model
- Default port: `11434`
**Step 3: Configure API Base**
```python
OPENAI_API_BASE = "http://localhost:11434/v1"
```
---
## 9. File Formats and Storage Locations
### Prompt Files (`prompts/`)
**Format:** Plain text (.txt)
**Naming Convention:** `model_<name>_final_prompt.txt`
**Example Content Structure:**
```
# Optimized Horror Text Adventure Prompt for Qwen-3.5-1B
[Instructions]
You are a horror text adventure generator...
[Specific Guidelines]
- Focus on sensory details (sight, sound, smell)
- Build tension gradually over 8 rooms
- Use atmospheric descriptions and pacing techniques
- Avoid repetition in phrasing while maintaining consistency
```
### Grading Feedback Files (`feedback/`)
**Format:** JSON (.json)
**Naming Convention:** `human_feedback_<model>.json`
**Structure:**
```json
{
"room_1": {
"room_content": "...",
"grades": {
"horror_quality": 4,
"coherence_score": 0.85,
"repetition_rate": 0.3,
"creativity_index": 4,
"narrative_flow": 4
},
"analysis": "Strong atmosphere but some repetitive phrasing..."
}
}
```
### Quirks Database (`quirks/`)
**Format:** JSON (.json)
**Naming Convention:** `model_<name>_quirks.json`
**Structure:**
```json
{
"<model_name>": {
"hallucinates_details": false,
"overuses_adjectives": true,
"short_responses": false,
"struggles_with_pacing": false
}
}
```
### Quirk Types Explained
| Quirk Type | Meaning | Impact on Optimization |
|------------|---------|------------------------|
| `hallucinates_details` | Model adds excessive detail | Add constraints to limit verbosity |
| `overuses_adjectives` | Too many descriptive words | Reduce adjective density in prompt |
| `short_responses` | Generates brief outputs | Increase token limits or add elaboration instructions |
| `struggles_with_pacing` | Poor narrative flow | Emphasize pacing and transition guidance |
---
## 10. Troubleshooting Common Issues
### Issue: "No golden prompt found"
**Symptoms:** Phase 1A fails with error message
**Solution:**
```bash
# Check if prompts directory exists
ls prompts/
# Create golden_prompt.txt if missing
echo "Your horror text adventure prompt here..." > prompts/golden_prompt.txt
```
### Issue: API Connection Failed
**Symptoms:** `LLM API request failed` error in console
**Solution:**
1. Verify LM Studio server is running (check port 1234)
2. Check `OPENAI_API_BASE` configuration matches your setup
3. Ensure API key is correctly set if required by proxy
4. Test connection manually:
```bash
curl http://localhost:1234/v1/models
```
### Issue: Model Not Found on API
**Symptoms:** Model marked with `[?]` or not available in validation list
**Solution:**
1. Verify model exists in LM Studio library
2. Ensure model is loaded/ready before running workflow
3. Update `MODELS` dictionary to match actual model names
4. Consider using a different model that's available on the server
### Issue: JSON Parsing Errors
**Symptoms:** `Failed to parse optimization response` or grading errors
**Solution:**
1. Check API response format matches expected structure
2. Verify teacher model is set correctly (should be qwen-3.5-9b)
3. Ensure proper temperature settings for different phases:
- Teacher optimization: moderate creativity (0.7)
- Grading: deterministic output
### Issue: Quirks Database Empty
**Symptoms:** "No pre-loaded quirks found" message
**Solution:**
1. Create quirks directory if missing: `mkdir quirks`
2. Add model-specific quirks JSON files to the directory
3. Use sample data from `quirks/model_quirks.json` as template
### Issue: Insufficient Grading Rooms
**Symptoms:** "Minimum rooms for grading not met" warning
**Solution:**
1. Increase `rooms_per_iteration` in `config/llm_api_config.py`
2. Ensure human grader has time to review all generated rooms
3. Consider reducing fine-tuning iterations if room generation is slow
### Issue: Repetitive Output Despite Optimization
**Symptoms:** Generated rooms still show high repetition rates
**Solution:**
1. Review quirks for "overuses_adjectives" or similar issues
2. Add explicit anti-repetition constraints to prompt
3. Increase number of fine-tuning iterations in Phase 1C
4. Consider using a different student model with different quirks
---
## Quick Reference Commands
```bash
# Initialize workflow (create golden prompt if needed)
python main.py
# Check prompts directory
ls -la prompts/
# View optimized prompt for specific model
cat prompts/model_qwen_3.5_1b_final_prompt.txt
# Review grading results
cat feedback/human_feedback_model_qwen_3.5_1b.json
# Test API connection
python llm/api_client.py
# Run single phase (requires modifications to main.py)
# Example: python -c "from main import WorkflowController; c = WorkflowController(); c.run_phase_2a()"
```
---
*Last Updated: 2026-06-15*
*System Version: 1.0*

View File

@@ -0,0 +1,98 @@
# Horror Prompt Optimization System
Iterative prompt optimization system for generating horror maze content via LLMs.
## Architecture
```
utils/ikeatraining/
├── config/
│ ├── llm_api_config.py # OpenAI API + LM Studio configuration
│ └── grading_config.py # Multi-dimensional scoring thresholds
├── data/
│ └── storage.py # JSON file I/O utilities
├── llm/
│ ├── api_client.py # Generic LLM API wrapper (OpenAI/LM Studio)
│ ├── grading_engine.py # Multi-dimensional room grading
│ └── teacher_optimizer.py # Teacher model for optimization
├── prompts/ # Optimized prompts per model variant
├── feedback/ # Human grading feedback JSON files
├── quirks/ # Model quirks database (optional)
├── tui/
│ └── status_display.py # Curses-based TUI display
├── main.py # Workflow controller
├── requirements.txt # Python dependencies
└── README.md # This file
```
## Configuration
### API Settings (`config/llm_api_config.py`)
- **API Base**: `http://localhost:1234/v1` (LM Studio default)
- **Model Selection**: 6 candidates from 0.5B to 9B parameter sizes
- **Temperature**: 0.7 for all models
### Grading Configuration (`config/grading_config.py`)
- **Rooms per iteration**: 8 rooms
- **Scoring dimensions**:
- Horror Quality (1-5)
- Coherence Score (0-1)
- Repetition Rate (0-1)
- Creativity Index (1-5)
- Narrative Flow (1-5)
## Workflow Phases
### Phase 1A: Golden Prompt Creation
Human writes baseline prompt based on horror text adventure expertise.
### Phase 1B: Teacher Coarse Optimization
Teacher model uses known quirks database to optimize the golden prompt.
### Phase 1C: Fine Tuning
Multiple iterations of fine-tuning against gold standard (8 rooms tested per iteration).
### Phase 2A: Human Grading
Human plays through adventure, grades rooms on 5 dimensions (8 rooms per model).
### Phase 2B: Teacher Iteration
Teacher uses human feedback to accelerate prompt optimization.
## Usage
```bash
# Install dependencies (in virtual environment)
venv\Scripts\pip.exe install -r requirements.txt
# Run the workflow
python main.py
```
**Model Selection:** The script will display available models and let you choose which one to test. Each model has different performance characteristics:
- **0.5B**: Fastest, but may struggle with coherence
- **1B**: Balanced speed/quality
- **2B-3.8B**: Better quality, slower generation
- **9B+**: Highest quality, slowest
## Environment Variables
- `OPENAI_API_BASE`: API base URL (default: http://localhost:1234/v1)
- `OPENAI_API_KEY`: Your API key for LM Studio/LiteLLM
- `DEFAULT_MODEL`: Default model name to use if no selection is made
## Testing with SearXNG MCP
For quirks database, you can prepare a markdown file using SearXNG MCP queries like:
```bash
# Example query format (would be run via MCP)
curl "https://searxng.randomhack.com/search?q=Qwen3.5+model+quirks"
```
## API Provider Options
- **OpenAI API** via LM Studio (`http://localhost:1234/v1`)
- **LiteLLM proxy** (can route to any compatible model)
- **HuggingFace Inference API** (alternative option)

View File

@@ -0,0 +1,89 @@
"""
Configuration for multi-dimensional room grading system
"""
# Grading Thresholds (pass/fail criteria)
GRADING_THRESHOLDS = {
"horror_quality": {"min_pass": 3, "max_fail": 1},
"coherence_score": {"min_pass": 0.7, "max_fail": 0.4},
"repetition_rate": {"min_pass": 0.2, "max_fail": 0.6},
"creativity_index": {"min_pass": 3, "max_fail": 1},
"narrative_flow": {"min_pass": 3, "max_fail": 1},
}
# Grading Scale Definitions
GRADING_SCALES = {
"horror_quality": {
"scale_name": "Horror Quality",
"description": "Rate the atmosphere, tension, and horror elements (1-5)",
"examples": {
1: "No horror elements, completely mundane",
2: "Weak horror attempt, feels forced",
3: "Decent horror atmosphere, some effective scares",
4: "Strong horror elements, well-executed tension",
5: "Masterful horror writing, immersive and terrifying"
}
},
"coherence_score": {
"scale_name": "Coherence Score",
"description": "Rate narrative consistency and logical flow (0-1)",
"examples": {
0.4: "Major plot holes, confusing connections",
0.7: "Mostly consistent with minor issues",
1.0: "Perfectly coherent, no contradictions"
}
},
"repetition_rate": {
"scale_name": "Repetition Rate",
"description": "Rate word/phrasing duplication (lower is better) (0-1)",
"examples": {
0.2: "Minimal repetition, fresh phrasing each time",
0.6: "Noticeable repetition patterns emerging",
1.0: "Highly repetitive, same phrases repeated"
}
},
"creativity_index": {
"scale_name": "Creativity Index",
"description": "Rate originality of horror elements (1-5)",
"examples": {
1: "Generic horror tropes, nothing unique",
3: "Some creative elements mixed with clichés",
4: "Strong original concepts, well-executed",
5: "Highly original, fresh and unexpected"
}
},
"narrative_flow": {
"scale_name": "Narrative Flow",
"description": "Rate pacing and transitions between rooms (1-5)",
"examples": {
1: "Abrupt jumps, confusing connections",
3: "Smooth but predictable progression",
4: "Well-paced with engaging transitions",
5: "Masterful flow, each room builds naturally"
}
},
}
# Grading Instructions for LLM
GRADING_INSTRUCTIONS = """You are an expert horror text adventure critic. Rate the following room content on five dimensions:
1. HORROR QUALITY (1-5): How effective is the horror atmosphere? Are there genuine scares, tension-building elements, or is it just generic creepiness?
2. COHERENCE SCORE (0-1): Is the narrative consistent? Do connections make sense? Are there plot holes or confusing elements?
3. REPETITION RATE (0-1): How repetitive are the word choices and phrasing compared to previous rooms? Lower is better - we want fresh horror each time.
4. CREATIVITY INDEX (1-5): How original are the horror elements? Are they generic tropes or genuinely creative and unexpected?
5. NARRATIVE FLOW (1-5): How well do transitions work between rooms? Is pacing good, or does it feel rushed/clunky?
Provide your ratings as JSON in this format:
{
"horror_quality": <1-5>,
"coherence_score": <0-1>,
"repetition_rate": <0-1>,
"creativity_index": <1-5>,
"narrative_flow": <1-5>
}
Be critical but fair. If something is truly exceptional, rate it high. If it's weak horror, don't be afraid to give low scores."""

View File

@@ -0,0 +1,56 @@
"""
Configuration for LLM API (OpenAI via LM Studio/LiteLLM)
"""
# OpenAI API Configuration
OPENAI_API_BASE = "http://localhost:1234/v1" # LM Studio default port
OPENAI_API_KEY = "your-api-key-here" # Replace with actual key or use LiteLLM proxy
# Model Configuration
MODELS = {
"qwen-3.5-0.5b": {"temperature": 0.7, "max_tokens": 2048},
"qwen-3.5-1b": {"temperature": 0.7, "max_tokens": 2048},
"gemma-3-2b": {"temperature": 0.7, "max_tokens": 2048},
"phi-3-mini-3.8b": {"temperature": 0.7, "max_tokens": 2048},
"qwen-3.5-9b": {"temperature": 0.7, "max_tokens": 4096}, # Teacher model
}
# API Configuration for LiteLLM/OpenAI
API_CONFIG = {
"model_name": "qwen-3.5-1b", # Default test model (smallest)
"temperature": 0.7,
"max_tokens": 2048,
"api_base": OPENAI_API_BASE,
"api_key": OPENAI_API_KEY,
}
# Grading Configuration
GRADING_CONFIG = {
"rooms_per_iteration": 8,
"iterations_per_phase": 3,
"min_rooms_for_grading": 5,
"scoring_dimensions": [
{"name": "horror_quality", "scale": (1, 5), "weight": 0.25},
{"name": "coherence_score", "scale": (0, 1), "weight": 0.2},
{"name": "repetition_rate", "scale": (0, 1), "weight": 0.15},
{"name": "creativity_index", "scale": (1, 5), "weight": 0.2},
{"name": "narrative_flow", "scale": (1, 5), "weight": 0.2},
],
}
# Phase Configuration
PHASE_CONFIG = {
"phase_1a_golden_prompt_iterations": 1, # Human writes once initially
"phase_1b_teacher_optimization_iterations": 3,
"phase_1c_fine_tuning_iterations": 5,
"phase_2a_human_grading_rooms": 8,
"phase_2b_teacher_iteration_iterations": 4,
}
# Output Configuration
OUTPUT_CONFIG = {
"prompt_directory": "prompts",
"feedback_directory": "feedback",
"quirks_directory": "quirks",
"data_storage_path": "data/storage.json",
}

View File

@@ -0,0 +1,149 @@
"""
Data Storage Utilities - JSON file I/O operations
"""
import json
from typing import Dict, List, Optional, Any
import os
class DataStorage:
"""Utility class for reading/writing JSON files"""
def __init__(self, storage_path: Optional[str] = None):
"""
Initialize data storage
Args:
storage_path: Path to the main storage file (default: data/storage.json)
"""
self.storage_path = storage_path or "data/storage.json"
def _ensure_directory(self, path: str):
"""Ensure parent directory exists"""
dir_path = os.path.dirname(path)
if not os.path.exists(dir_path):
os.makedirs(dir_path)
def read_json(self, filepath: str) -> dict:
"""Read and parse a JSON file"""
self._ensure_directory(filepath)
with open(filepath, 'r', encoding='utf-8') as f:
return json.load(f)
def write_json(self, filepath: str, data: dict):
"""Write data to a JSON file"""
self._ensure_directory(filepath)
with open(filepath, 'w', encoding='utf-8') as f:
json.dump(data, f, indent=2, ensure_ascii=False)
def read_prompts(self) -> Dict[str, str]:
"""Read all prompts from the prompts directory"""
prompts = {}
if not os.path.exists("prompts"):
return prompts
for filename in os.listdir("prompts"):
if filename.endswith('.txt'):
filepath = "prompts/" + filename
with open(filepath, 'r', encoding='utf-8') as f:
content = f.read()
# Extract model name from filename (e.g., "model_qwen_3.5_1b_final_prompt.txt")
model_name = filename.replace('.txt', '').replace('_', ' ').lower()
prompts[model_name] = content
return prompts
def write_prompts(self, data: Dict[str, str]):
"""Write prompts to files in the prompts directory"""
for model_name, prompt_content in data.items():
filepath = "prompts/model_" + model_name + "_final_prompt.txt"
with open(filepath, 'w', encoding='utf-8') as f:
f.write(prompt_content)
def read_feedback(self, model_name: str = "") -> Dict[str, Any]:
"""Read feedback for a specific model"""
if not model_name: # Empty string means "all models"
# Read all feedback files
feedback = {}
if not os.path.exists("feedback"):
return feedback
for filename in os.listdir("feedback"):
if filename.endswith('.json'):
filepath = "feedback/" + filename
data = self.read_json(filepath)
# Extract model name from filename
base_name = filename.replace('.json', '')
model_key = base_name.lower().replace('_', ' ')
feedback[model_key] = data
return feedback
else:
filepath = "feedback/human_feedback_" + model_name + ".json"
if not os.path.exists(filepath):
return {}
return self.read_json(filepath)
def write_feedback(self, model_name: str, data: Dict[str, Any]):
"""Write feedback for a specific model"""
filepath = "feedback/human_feedback_" + model_name + ".json"
with open(filepath, 'w', encoding='utf-8') as f:
json.dump(data, f, indent=2, ensure_ascii=False)
def read_quirks(self) -> Dict[str, Any]:
"""Read quirks database"""
if not os.path.exists("quirks"):
return {}
quirks = {}
for filename in os.listdir("quirks"):
if filename.endswith('.json'):
filepath = "quirks/" + filename
data = self.read_json(filepath)
# Extract model name from filename
base_name = filename.replace('.json', '')
model_key = base_name.lower().replace('_', ' ')
quirks[model_key] = data
return quirks
def write_quirks(self, data: Dict[str, Any]):
"""Write quirks database"""
for model_name, quirks_data in data.items():
filepath = "quirks/model_" + model_name + "_quirks.json"
with open(filepath, 'w', encoding='utf-8') as f:
json.dump(quirks_data, f, indent=2, ensure_ascii=False)
# Global storage instance
_storage_instance = None
def get_storage() -> DataStorage:
"""Get or create the global data storage instance"""
global _storage_instance
if _storage_instance is None:
_storage_instance = DataStorage("data/storage.json")
return _storage_instance
if __name__ == "__main__":
# Test the storage utility
storage = get_storage()
print(f"Storage initialized with path: {storage.storage_path}")

View File

@@ -0,0 +1,142 @@
"""
Generic LLM API Client for OpenAI-compatible APIs (via LM Studio/LiteLLM)
"""
import json
import os
from typing import Dict, List, Optional, Any
from datetime import datetime
class LLMAPIError(Exception):
"""Custom exception for LLM API errors"""
pass
class LLMAPIClient:
"""Client for OpenAI-compatible APIs (OpenAI, LM Studio, LiteLLM)"""
def __init__(self, api_base: str, api_key: str, model_name: Optional[str] = None):
"""
Initialize LLM API client
Args:
api_base: Base URL for the API (e.g., http://localhost:1234/v1)
api_key: API key for authentication
model_name: Default model name to use
"""
self.api_base = api_base.rstrip('/') + '/'
self.api_key = api_key
self.model_name = model_name or "default-model"
def _make_request(self, endpoint: str, params: Dict) -> Dict:
"""Make a request to the API"""
import requests
url = f"{self.api_base}{endpoint}"
headers = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
}
response = requests.post(url, json=params, headers=headers)
if response.status_code >= 400:
error_msg = response.json().get("error", {}).get("message", f"API Error {response.status_code}")
raise LLMAPIError(f"API request failed: {error_msg}")
return response.json()
def chat_completion(self, messages: List[Dict], model_name: Optional[str] = None) -> Dict:
"""
Make a chat completion request
Args:
messages: List of message objects with 'role' and 'content' keys
model_name: Optional specific model to use
Returns:
Dictionary containing the response content
"""
if model_name is None and self.model_name:
model_name = self.model_name
params = {
"model": model_name or "default-model",
"messages": messages,
"temperature": 0.7,
"max_tokens": 2048,
}
response = self._make_request("chat/completions", params)
# Handle different API response formats (OpenAI vs LiteLLM/LM Studio)
content = ""
api_model = model_name
try:
# Try standard OpenAI format first
choices = response.get("choices", [])
if choices and len(choices) > 0:
message = choices[0].get("message", {})
content = message.get("content", "")
api_model = model_name # Use the requested model name
except (KeyError, IndexError):
pass
return {
"content": content,
"model": api_model,
}
def generate(self, prompt: str, model_name: Optional[str] = None) -> Dict:
"""
Make a text generation request
Args:
prompt: The input prompt/text to generate from
model_name: Optional specific model to use
Returns:
Dictionary containing the generated content
"""
if model_name is None and self.model_name:
model_name = self.model_name
messages = [
{"role": "user", "content": prompt}
]
response = self.chat_completion(messages, model_name or "default-model")
return {
"content": response["content"],
"model": response["model"],
}
def save_response(self, content: str, filename: str):
"""Save generated content to a file"""
filepath = f"{filename}.txt"
with open(filepath, 'w', encoding='utf-8') as f:
f.write(content)
# Global client instance (can be configured via environment variables)
def get_client() -> LLMAPIClient:
"""Get or create the global LLM API client"""
api_base = os.getenv("OPENAI_API_BASE", "http://localhost:1234/v1")
api_key = os.getenv("OPENAI_API_KEY", "")
# Try to detect model from environment or use default
model_name = os.getenv("DEFAULT_MODEL", None)
return LLMAPIClient(api_base, api_key, model_name)
if __name__ == "__main__":
# Test the client
client = get_client()
print(f"Client initialized with base: {client.api_base}")
print(f"Model: {client.model_name}")

View File

@@ -0,0 +1,150 @@
"""
Multi-Dimensional Room Grading Engine for Horror Text Adventure Content
"""
import json
from typing import Dict, List, Any
from datetime import datetime
GRADING_INSTRUCTIONS = """You are an expert horror text adventure critic. Rate the following room content on five dimensions:
1. HORROR QUALITY (1-5): How effective is the horror atmosphere? Are there genuine scares, tension-building elements, or is it just generic creepiness?
2. COHERENCE SCORE (0-1): Is the narrative consistent? Do connections make sense? Are there plot holes or confusing elements?
3. REPETITION RATE (0-1): How repetitive are the word choices and phrasing compared to previous rooms? Lower is better - we want fresh horror each time.
4. CREATIVITY INDEX (1-5): How original are the horror elements? Are they generic tropes or genuinely creative and unexpected?
5. NARRATIVE FLOW (1-5): How well do transitions work between rooms? Is pacing good, or does it feel rushed/clunky?
Provide your ratings as JSON in this format:
{
"horror_quality": <1-5>,
"coherence_score": <0-1>,
"repetition_rate": <0-1>,
"creativity_index": <1-5>,
"narrative_flow": <1-5>
}
Be critical but fair. If something is truly exceptional, rate it high. If it's weak horror, don't be afraid to give low scores."""
class RoomGrader:
"""Handles multi-dimensional grading of horror room content"""
def __init__(self, api_client):
self.api_client = api_client
def grade_room(self, room_content: str, previous_rooms: List[str]) -> Dict:
messages = [
{
"role": "system",
"content": GRADING_INSTRUCTIONS
},
{
"role": "user",
"content": self._build_grading_prompt(room_content, previous_rooms)
}
]
response = self.api_client.chat_completion(messages)
try:
grades = json.loads(response["content"])
return {
"room_content": room_content[:500] + "...",
"grades": grades,
"analysis": self._parse_analysis(response["content"]),
"timestamp": datetime.now().isoformat(),
}
except json.JSONDecodeError:
return {
"room_content": room_content[:500] + "...",
"error": "Failed to parse grading response",
"analysis": self._parse_analysis(response["content"]),
"timestamp": datetime.now().isoformat(),
}
def _build_grading_prompt(self, room_content: str, previous_rooms: List[str]) -> str:
context = ""
if len(previous_rooms) > 0:
context = "Previous rooms:\n" + "\n".join([f"{i+1}. {r[:200]}..." for i, r in enumerate(previous_rooms)])
return f"""Grade the following room content on these dimensions:
{context}
Room Content to Grade:
{'=' * 50}
{room_content}
{'=' * 50}
Provide your ratings as JSON with this exact format:
{{
"horror_quality": <1-5>,
"coherence_score": <0-1>,
"repetition_rate": <0-1>,
"creativity_index": <1-5>,
"narrative_flow": <1-5>
}}
Include brief analysis for each dimension in the response."""
def _parse_analysis(self, content: str) -> Dict[str, Any]:
try:
json_end = content.find("}")
if json_end != -1:
json_part = content[:json_end]
analysis_part = content[json_end + 2:]
return {
"analysis": analysis_part.strip(),
"raw_response": content
}
except Exception:
pass
return {"analysis": content, "raw_response": ""}
class BatchGrader(RoomGrader):
"""Handles grading multiple rooms in batch"""
def __init__(self, api_client, max_concurrent_requests: int = 1):
super().__init__(api_client)
self.max_concurrent_requests = max_concurrent_requests
def grade_multiple_rooms(self, room_contents: List[str]) -> List[Dict]:
grades = []
for i, content in enumerate(room_contents):
if i > 0 and len(grades) > 0:
previous_rooms = [g["room_content"] for g in grades]
else:
previous_rooms = []
result = self.grade_room(content, previous_rooms)
grades.append(result)
return grades
def grade_rooms_for_testing(api_client, room_contents: List[str]) -> List[Dict]:
grader = BatchGrader(api_client)
return grader.grade_multiple_rooms(room_contents)
if __name__ == "__main__":
from llm.api_client import get_client
client = get_client()
grader = RoomGrader(client)
test_room = """You enter a dimly lit hallway. The walls are covered in peeling paint, revealing dark wood underneath. A flickering fluorescent light buzzes overhead. You hear faint scratching sounds coming from the shadows ahead."""
result = grader.grade_room(test_room, [])
print(json.dumps(result, indent=2))

View File

@@ -0,0 +1,174 @@
"""
Teacher Optimizer - Uses known model quirks (via SearXNG) to optimize prompts
"""
import json
from typing import Dict, List, Optional, Any
from datetime import datetime
class TeacherOptimizer:
"""Optimizes prompts using teacher model and known quirks database"""
def __init__(self, api_client, quirks_data: Dict[str, Any] = {}):
"""
Initialize teacher optimizer
Args:
api_client: LLM API client instance (teacher model)
quirks_data: Optional pre-loaded quirks data from SearXNG
"""
self.api_client = api_client
self.quirks_data = quirks_data or {}
def optimize_prompt(self, original_prompt: str, target_model_name: str,
human_feedback: Dict[str, Any] = {}) -> Dict:
"""
Optimize a prompt using the teacher model and known quirks
Args:
original_prompt: The original golden prompt to optimize
target_model_name: Name of the model we're optimizing for
human_feedback: Optional human feedback data
Returns:
Dictionary containing optimized prompt and analysis
"""
# Build context with quirks and feedback
context = self._build_optimization_context(original_prompt, target_model_name,
human_feedback)
# Create messages for optimization request
messages = [
{
"role": "system",
"content": self._get_teacher_system_prompt()
},
{
"role": "user",
"content": context
}
]
# Get optimized prompt from teacher model
response = self.api_client.chat_completion(messages)
try:
optimized_content = json.loads(response["content"])
return {
"original_prompt": original_prompt[:500] + "...",
"optimized_prompt": optimized_content,
"analysis": self._parse_analysis(response["content"]),
"model_quirks_applied": target_model_name in self.quirks_data,
"timestamp": datetime.now().isoformat(),
}
except json.JSONDecodeError:
return {
"original_prompt": original_prompt[:500] + "...",
"optimized_prompt": response["content"], # Use raw content if JSON fails
"error": "Failed to parse optimization response",
"analysis": self._parse_analysis(response["content"]),
"model_quirks_applied": target_model_name in self.quirks_data,
"timestamp": datetime.now().isoformat(),
}
def _build_optimization_context(self, original_prompt: str, target_model_name: str,
human_feedback: Dict[str, Any]) -> str:
"""Build the context for prompt optimization"""
context_parts = []
# Add model quirks if available
if target_model_name in self.quirks_data and self.quirks_data[target_model_name]:
quirks = self.quirks_data[target_model_name]
quirks_text = "Known quirks for this model:\n"
for quirk_type, details in quirks.items():
quirks_text += f"- {quirk_type}: {details}\n"
context_parts.append(quirks_text)
# Add human feedback if available
if human_feedback and len(human_feedback) > 0:
feedback_text = "Human grading feedback:\n"
for room_num, data in human_feedback.items():
grades = data.get("grades", {})
feedback_text += f"\nRoom {room_num}:\n"
# Add key issues based on scores
if grades.get("horror_quality", 5) < 3:
feedback_text += " - Horror quality too low\n"
if grades.get("creativity_index", 5) < 3:
feedback_text += " - Creativity needs improvement\n"
if grades.get("narrative_flow", 5) < 3:
feedback_text += " - Narrative flow issues\n"
context_parts.append(feedback_text)
# Add original prompt
context_parts.append("\nOriginal Prompt to Optimize:")
context_parts.append("=" * 60)
context_parts.append(original_prompt)
context_parts.append("=" * 60)
return "\n".join(context_parts)
def _get_teacher_system_prompt(self) -> str:
"""Get the system prompt for the teacher model"""
return """You are an expert horror text adventure writer and prompt optimizer.
Your task is to optimize prompts for generating horror maze content.
When optimizing, consider:
1. Known model quirks (provided in context if available)
2. Human feedback on previous iterations (provided in context if available)
3. The specific requirements of the target model
Focus on improving:
- Horror atmosphere and tension
- Originality and creativity
- Narrative flow and coherence
- Avoiding repetition while maintaining consistency
Output your optimized prompt as valid JSON with this structure:
{{
"optimized_prompt": "<the full optimized prompt>",
"improvements_made": ["<list of key improvements>"],
"rationale": "<brief explanation of changes>"
}}"""
def _parse_analysis(self, content: str) -> Dict[str, Any]:
"""Parse and extract analysis from optimization response"""
try:
# Try to find JSON in the response
json_end = content.find("}")
if json_end != -1:
json_part = content[:json_end]
analysis_part = content[json_end + 2:] # Skip "}" and newline
return {
"analysis": analysis_part.strip(),
"raw_response": content
}
except Exception:
pass
return {"analysis": content, "raw_response": ""}
def get_teacher_optimizer(api_client, quirks_data: Dict[str, Any] = {}) -> TeacherOptimizer:
"""Factory function to create teacher optimizer instance"""
return TeacherOptimizer(api_client, quirks_data)
if __name__ == "__main__":
# Test the optimizer
from llm.api_client import get_client
client = get_client()
optimizer = TeacherOptimizer(client)
test_prompt = """You are a horror text adventure generator. Create immersive and terrifying maze rooms with strong atmosphere."""
result = optimizer.optimize_prompt(test_prompt, "qwen-3.5-1b")
print(json.dumps(result, indent=2))

305
utils/ikeatraining/main.py Normal file
View File

@@ -0,0 +1,305 @@
"""
Main Workflow Controller - Orchestrates the prompt optimization process
"""
import json
from datetime import datetime
from typing import Dict, List, Any, Optional
# Import configuration
from config.llm_api_config import API_CONFIG, MODELS, GRADING_CONFIG, PHASE_CONFIG
from config.grading_config import GRADING_THRESHOLDS
class WorkflowController:
"""Main controller for the prompt optimization workflow"""
def __init__(self):
self.api_client = None
self.storage = None
self.quirks_data = {}
# Current state
self.current_phase = None
self.current_model_name = None
self.current_iteration = 0
# Data storage
self.golden_prompt = ""
self.optimized_prompts = {}
self.human_feedback = {}
def initialize(self):
"""Initialize all components"""
from llm.api_client import get_client
from data.storage import get_storage
self.api_client = get_client()
self.storage = get_storage()
# Load quirks database (from file or empty)
self.quirks_data = self.storage.read_quirks()
def run_phase_1a(self):
"""Phase 1A: Human writes golden prompt"""
print("\n" + "=" * 60)
print("PHASE 1A: HUMAN WRITES GOLDEN PROMPT")
print("=" * 60)
# Read golden prompt from file
prompts = self.storage.read_prompts() if self.storage else {}
self.golden_prompt = prompts.get("golden", "No golden prompt found")
if not self.golden_prompt:
print("\nERROR: No golden prompt found in prompts/golden_prompt.txt")
print("Please create the file first before running the workflow.")
exit(1)
print(f"\nGolden Prompt loaded:")
print(self.golden_prompt)
def run_phase_1b(self):
"""Phase 1B: Teacher coarse optimization using quirks"""
from llm.teacher_optimizer import get_teacher_optimizer
self.current_model_name = API_CONFIG["model_name"]
# Get quirks for this model
if self.current_model_name in self.quirks_data:
quirks = self.quirks_data[self.current_model_name]
print(f"\nModel quirks loaded for {self.current_model_name}")
for quirk_type, details in quirks.items():
print(f" - {quirk_type}: {details}")
else:
print(f"\nNo pre-loaded quirks found for {self.current_model_name}")
# Optimize prompt using teacher model
optimizer = get_teacher_optimizer(self.api_client, self.quirks_data)
result = optimizer.optimize_prompt(
self.golden_prompt,
self.current_model_name,
{} # No human feedback yet in Phase 1B
)
print(f"\nOptimization complete for {self.current_model_name}")
print(f"Model quirks applied: {result['model_quirks_applied']}")
# Save optimized prompt
self.optimized_prompts[self.current_model_name] = result["optimized_prompt"]
def run_phase_1c(self):
"""Phase 1C: Fine tuning - multiple iterations"""
from llm.teacher_optimizer import get_teacher_optimizer
print("\n" + "=" * 60)
print("PHASE 1C: FINE TUNING")
print("=" * 60)
# Get list of models to fine-tune
model_names = [name for name in MODELS.keys() if name != "qwen-3.5-9b"] # Exclude teacher
total_iterations = PHASE_CONFIG["phase_1c_fine_tuning_iterations"]
for i, model_name in enumerate(model_names):
print(f"\n--- Fine-tuning {model_name} (iteration {i+1}/{total_iterations}) ---")
# Get quirks for this model
if model_name in self.quirks_data:
quirks = self.quirks_data[model_name]
optimizer = get_teacher_optimizer(self.api_client, self.quirks_data)
result = optimizer.optimize_prompt(
self.golden_prompt,
model_name,
{} # No human feedback yet in Phase 1C
)
print(f"Optimized prompt for {model_name}")
self.optimized_prompts[model_name] = result["optimized_prompt"]
else:
print(f"No quirks found for {model_name}, skipping")
def run_phase_2a(self):
"""Phase 2A: Human grading - test rooms"""
from llm.grading_engine import grade_rooms_for_testing
self.current_model_name = API_CONFIG["model_name"]
print("\n" + "=" * 60)
print("PHASE 2A: HUMAN GRADING")
print("=" * 60)
# Generate test rooms (simulated - in real implementation, would generate from prompt)
num_rooms = GRADING_CONFIG["rooms_per_iteration"]
# Simulate room content generation
test_rooms = [f"Test room {i+1} generated from optimized prompt for {self.current_model_name}"
for i in range(num_rooms)]
print(f"\nGenerating {num_rooms} test rooms...")
# Grade the rooms
room_grades = grade_rooms_for_testing(self.api_client, test_rooms)
print("\nGrading results:")
for i, result in enumerate(room_grades):
if "error" not in result:
grades = result["grades"]
print(f"\nRoom {i+1}:")
print(f" Horror Quality: {grades['horror_quality']}/5")
print(f" Coherence Score: {grades['coherence_score']}")
print(f" Repetition Rate: {grades['repetition_rate']}")
print(f" Creativity Index: {grades['creativity_index']}/5")
print(f" Narrative Flow: {grades['narrative_flow']}/5")
# Save feedback to storage
self.human_feedback = {}
for i, result in enumerate(room_grades):
if "error" not in result:
grades = result["grades"]
self.human_feedback[str(i+1)] = {
"room_content": result["room_content"],
"grades": grades,
"analysis": result.get("analysis", {}).get("analysis", "")
}
# Save to file
with open("feedback/human_feedback.json", 'w') as f:
json.dump(self.human_feedback, f, indent=2)
def run_phase_2b(self):
"""Phase 2B: Teacher iteration using human feedback"""
from llm.teacher_optimizer import get_teacher_optimizer
print("\n" + "=" * 60)
print("PHASE 2B: TEACHER ITERATION")
print("=" * 60)
# Get list of models to iterate on
model_names = [name for name in MODELS.keys() if name != "qwen-3.5-9b"]
total_iterations = PHASE_CONFIG["phase_2b_teacher_iteration_iterations"]
for i, model_name in enumerate(model_names):
print(f"\n--- Iterating {model_name} (iteration {i+1}/{total_iterations}) ---")
# Get quirks and human feedback for this model
if model_name in self.quirks_data:
quirks = self.quirks_data[model_name]
optimizer = get_teacher_optimizer(self.api_client, self.quirks_data)
result = optimizer.optimize_prompt(
self.golden_prompt,
model_name,
self.human_feedback # Use human feedback from Phase 2A
)
print(f"Optimized prompt for {model_name}")
self.optimized_prompts[model_name] = result["optimized_prompt"]
else:
print(f"No quirks found for {model_name}, skipping")
def main():
"""Main entry point"""
# Get API client with user-selected model
from llm.api_client import get_client
# Ask user for model selection
print("\n" + "=" * 60)
print("SELECT MODEL TO TEST")
print("=" * 60)
available_models = list(MODELS.keys())
if not available_models:
print("No models configured!")
exit(1)
# Fetch available models from API to validate selection
try:
client = get_client()
# Try to fetch available models from API
import requests
url = f"{client.api_base.rstrip('/')}/models"
response = requests.get(url, headers={"Authorization": f"Bearer {API_CONFIG['api_key']}"})
if response.status_code == 200:
api_models = response.json().get("data", [])
model_ids = [m["id"] for m in api_models]
# Filter available models to only those present on API
valid_models = []
for model_name in available_models:
if model_name in model_ids or any(model_id.startswith(model_name) for model_id in model_ids):
valid_models.append(model_name)
print(f"\nFound {len(valid_models)} models on API")
# Show only valid models
display_models = valid_models
else:
print(f"\nWarning: Could not fetch models from API (status {response.status_code})")
print("Using configured models list...")
display_models = available_models
except Exception as e:
print(f"\nWarning: Could not connect to API ({e})")
print("Using configured models list...")
display_models = available_models
print("\nAvailable models:")
for i, model in enumerate(display_models):
config = MODELS[model]
# Check if this model is on the API
api_available = model in valid_models or (valid_models and len(valid_models) > 0)
status = "[OK]" if api_available else "[?]"
print(f" {i+1}. {model} ({status}) temp={config['temperature']}, max_tokens={config['max_tokens']}")
try:
choice = int(input("\nEnter your choice (1-{}): ".format(len(display_models))) or 0)
except ValueError:
print("Invalid input, using default model...")
choice = 0
if choice == 0:
selected_model = API_CONFIG["model_name"]
else:
selected_model = display_models[choice - 1]
# Update API config with selected model
API_CONFIG["model_name"] = selected_model
controller = WorkflowController()
# Initialize components
controller.initialize()
# Run all phases
controller.run_phase_1a()
controller.run_phase_1b()
controller.run_phase_1c()
controller.run_phase_2a()
controller.run_phase_2b()
print("\n" + "=" * 60)
print("WORKFLOW COMPLETE")
print("=" * 60)
# Save all prompts to files
from data.storage import get_storage
storage = get_storage()
storage.write_prompts(controller.optimized_prompts)
print(f"\nOptimized prompts saved to 'prompts/' directory")
if __name__ == "__main__":
main()

View File

@@ -0,0 +1 @@
You are an expert horror text adventure writer specializing in maze and labyrinth settings. Your task is to generate immersive, terrifying room descriptions that build tension through atmosphere rather than jump scares. Each room should feel distinct from the previous one while maintaining narrative coherence. Focus on sensory details: dim lighting, unsettling sounds, oppressive air quality, and psychological dread. Avoid repetition of phrases or imagery across rooms. Create original horror concepts for each location.

View File

@@ -0,0 +1,26 @@
{
"qwen-3.5-0.5b": {
"hallucinates_details": true,
"overuses_adjectives": true,
"short_responses": false,
"struggles_with_pacing": true
},
"qwen-3.5-1b": {
"hallucinates_details": false,
"overuses_adjectives": true,
"short_responses": false,
"struggles_with_pacing": false
},
"gemma-3-2b": {
"hallucinates_details": false,
"overuses_adjectives": false,
"short_responses": true,
"struggles_with_pacing": false
},
"phi-3-mini-3.8b": {
"hallucinates_details": false,
"overuses_adjectives": false,
"short_responses": false,
"struggles_with_pacing": true
}
}

View File

@@ -0,0 +1,4 @@
requests>=2.31.0
# Optional: TUI display (not required for initial testing)
# curses>=6.0.0 # Windows may need alternative terminal handling

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,38 @@
"""Test LLM API Connection"""
import os
from llm.api_client import get_client, LLMAPIError
def main():
print("Testing LLM API connection...")
try:
client = get_client()
print(f"Client initialized:")
print(f" - Base URL: {client.api_base}")
print(f" - Model: {client.model_name}")
# Test with a simple prompt
test_prompt = "Hello, this is a test."
print(f"\nSending test request...")
result = client.generate(test_prompt)
print(f"\nResponse received:")
print(f" - Content: {result['content'][:100]}...")
print(f" - Model used: {result['model']}")
print("\n✓ API connection successful!")
except LLMAPIError as e:
print(f"\n✗ API Error: {e}")
print("Make sure LM Studio is running at http://localhost:1234/v1")
return 1
except Exception as e:
print(f"\n✗ Unexpected error: {e}")
return 1
return 0
if __name__ == "__main__":
exit(main())

View File

@@ -0,0 +1,139 @@
"""
TUI Status Display - Curses-based terminal UI for tracking optimization progress
"""
import curses
from typing import Dict, List, Any
class TUIDisplay:
"""Terminal UI display for horror prompt optimization workflow"""
def __init__(self, stdscr):
self.stdscr = stdscr
def render(self, phase: str, current_iteration: int, total_iterations: int,
model_name: str, progress_data: Dict[str, Any]):
"""Render the TUI display"""
# Clear screen
self.stdscr.clear()
# Title
title = "HORROR PROMPT OPTIMIZATION SYSTEM"
self.stdscr.addstr(0, 0, title.center(80), curses.A_BOLD)
self.stdscr.addstr(1, 0, "-" * 80)
# Current model display (moved to top)
model_info = f"Model: {model_name}"
self.stdscr.addstr(3, 0, model_info.ljust(40))
# Phase info
phase_info = f"PHASE: {phase}"
self.stdscr.addstr(5, 0, phase_info.ljust(40))
# Progress bar
progress_text = f"Iteration {current_iteration}/{total_iterations}"
self.stdscr.addstr(5, 0, progress_text)
# Calculate progress percentage
if total_iterations > 0:
progress_pct = (current_iteration / total_iterations) * 100
else:
progress_pct = 0
bar_width = 40
filled_len = int(bar_width * progress_pct / 100)
# Draw progress bar
self.stdscr.addstr(6, 0, "[" + "=" * filled_len + "-" * (bar_width - filled_len) + "]")
# Model info
model_info = f"Model: {model_name}"
self.stdscr.addstr(8, 0, model_info.ljust(40))
# Progress data display
if progress_data and "current_scores" in progress_data:
scores = progress_data["current_scores"]
self.stdscr.addstr(10, 0, "-" * 40)
self.stdscr.addstr(10, 0, f"HORROR QUALITY: {scores.get('horror_quality', '?'):>6}")
self.stdscr.addstr(11, 0, f"CREATIVITY INDEX: {scores.get('creativity_index', '?'):>6}")
self.stdscr.addstr(12, 0, f"NARRATIVE FLOW: {scores.get('narrative_flow', '?'):>6}")
if "coherence_score" in scores:
self.stdscr.addstr(13, 0, f"COHERENCE SCORE: {scores['coherence_score']:>8.2f}")
# Status messages
status_msg = progress_data.get("status", "")
if status_msg:
self.stdscr.addstr(15, 0, status_msg)
# Footer
footer = f"Press 'q' to quit | Press Enter for next iteration"
self.stdscr.addstr(self.stdscr.getmaxyx()[1] - 2, 0, footer)
self.stdscr.refresh()
def render_phase_1a(self, golden_prompt: str):
"""Render Phase 1A - Golden Prompt display"""
self.render("PHASE 1A", 1, 1, "Human Writer", {
"status": f"Writing golden prompt for {golden_prompt[:50]}...",
"current_scores": {}
})
def render_phase_1b(self, model_name: str, quirks_applied: bool):
"""Render Phase 1B - Teacher Optimization display"""
self.render("PHASE 1B", 2, 3, model_name, {
"status": f"Teacher optimizing with quirks: {'Yes' if quirks_applied else 'No'}",
"current_scores": {}
})
def render_phase_1c(self, iteration: int, total_iterations: int):
"""Render Phase 1C - Fine Tuning display"""
self.render("PHASE 1C", iteration + 1, total_iterations, "Fine-tuning Model", {
"status": f"Fine-tuning iteration {iteration}...",
"current_scores": {}
})
def render_phase_2a(self, model_name: str, rooms_graded: int):
"""Render Phase 2A - Human Grading display"""
self.render("PHASE 2A", rooms_graded + 1, 8, model_name, {
"status": f"Human grading {rooms_graded} rooms...",
"current_scores": {}
})
def render_phase_2b(self, iteration: int, total_iterations: int):
"""Render Phase 2B - Teacher Iteration display"""
self.render("PHASE 2B", iteration + 1, total_iterations, "Teacher Model", {
"status": f"Teacher iteration {iteration}...",
"current_scores": {}
})
def main(stdscr):
"""Main TUI loop"""
display = TUIDisplay(stdscr)
# Set curses options
curses.curs_set(0) # Hide cursor
stdscr.nodelay(True) # Non-blocking input
while True:
try:
key = stdscr.getch()
if key == ord('q') or key == ord('Q'):
break
elif key == ord('\n') or key == ord('\r'):
# Next iteration handler would go here
pass
except curses.error:
pass
if __name__ == "__main__":
curses.wrapper(main)