Complete toxicity analysis system setup and testing

- Copy and integrate analysis templates from Bluesky collector
- Add template filters (format_number, time_ago, truncate_text)
- Add Analysis link to navigation
- Fix analyzer database schema compatibility (account_db_id, status_type)
- Add OPENAI_API_KEY to docker-compose environment
- Successfully tested analyzer on 100 statuses ($0.0116, 75.4 seconds)

Web interface available at /analysis and /analysis/flagged

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Pieter 2026-03-30 15:39:36 +02:00
parent 27582c7b77
commit 2ff5ed78da
6 changed files with 1575 additions and 2 deletions

View file

@ -57,11 +57,11 @@ class AnalyzerDB:
Skips boosts (reblogs) and statuses with empty content.
"""
query = """
SELECT s.id, s.content, s.account_id
SELECT s.id, s.content, s.account_db_id
FROM statuses s
LEFT JOIN toxicity_scores ts ON ts.status_id = s.id
WHERE ts.status_id IS NULL
AND s.reblog_of_id IS NULL
AND s.status_type != 'reblog'
AND s.content IS NOT NULL
AND s.content != ''
ORDER BY s.created_at DESC

734
app/templates/analysis.html Normal file
View file

@ -0,0 +1,734 @@
{% extends "base.html" %}
{% block title %}Toxicity Analysis Dashboard{% endblock %}
{% block extra_css %}
<style>
/* Color scheme */
:root {
--bg-primary: #1a1a2e;
--bg-secondary: #16213e;
--nav-bg: #0f3460;
--text-primary: #e0e0e0;
--text-secondary: #b0b0b0;
--accent: #00b4d8;
--danger: #e74c3c;
--warning: #f39c12;
--success: #27ae60;
--category-1: #00b4d8;
--category-2: #e67e22;
--category-3: #9b59b6;
--category-4: #1abc9c;
--category-5: #e74c3c;
--category-6: #f39c12;
--category-7: #3498db;
--category-8: #2ecc71;
}
/* Layout */
.page-header {
margin-bottom: 2rem;
}
.page-header h1 {
font-size: 2.5rem;
font-weight: 700;
color: var(--text-primary);
margin-bottom: 0.5rem;
}
.page-header .subtitle {
font-size: 1rem;
color: var(--text-secondary);
}
/* Grid layout */
.stats-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
gap: 1.5rem;
margin-bottom: 2rem;
}
/* Stat cards */
.stat-card {
background-color: var(--bg-secondary);
border-radius: 8px;
padding: 1.5rem;
border-left: 4px solid var(--accent);
display: flex;
flex-direction: column;
justify-content: space-between;
}
.stat-card.danger {
border-left-color: var(--danger);
}
.stat-card.warning {
border-left-color: var(--warning);
}
.stat-card-label {
font-size: 0.875rem;
color: var(--text-secondary);
text-transform: uppercase;
letter-spacing: 0.5px;
margin-bottom: 0.75rem;
}
.stat-card-value {
font-size: 1.75rem;
font-weight: 700;
color: var(--text-primary);
margin-bottom: 0.5rem;
}
.stat-card-detail {
font-size: 0.875rem;
color: var(--text-secondary);
}
/* Percentage bar */
.percentage-bar {
width: 100%;
height: 8px;
background-color: rgba(224, 224, 224, 0.1);
border-radius: 4px;
overflow: hidden;
margin-top: 0.75rem;
}
.percentage-bar-fill {
height: 100%;
background-color: var(--accent);
border-radius: 4px;
transition: width 0.3s ease;
}
.stat-card.danger .percentage-bar-fill {
background-color: var(--danger);
}
/* Cards */
.card {
background-color: var(--bg-secondary);
border-radius: 8px;
padding: 1.5rem;
margin-bottom: 2rem;
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.3);
}
.card-title {
font-size: 1.25rem;
font-weight: 600;
color: var(--text-primary);
margin-bottom: 1.5rem;
border-bottom: 2px solid rgba(0, 180, 216, 0.2);
padding-bottom: 0.75rem;
}
/* Chart containers */
.chart-container {
position: relative;
height: 400px;
width: 100%;
}
.chart-container.horizontal {
height: 300px;
}
/* Links section */
.quick-links {
display: flex;
gap: 1rem;
flex-wrap: wrap;
}
.quick-link {
display: inline-flex;
align-items: center;
padding: 0.75rem 1.25rem;
background-color: rgba(0, 180, 216, 0.1);
border: 1px solid var(--accent);
border-radius: 6px;
color: var(--accent);
text-decoration: none;
font-weight: 500;
transition: all 0.3s ease;
}
.quick-link:hover {
background-color: var(--accent);
color: var(--bg-primary);
}
.quick-link::after {
content: " →";
margin-left: 0.5rem;
}
/* Runs table */
.runs-table {
width: 100%;
border-collapse: collapse;
}
.runs-table thead {
background-color: rgba(0, 180, 216, 0.1);
border-bottom: 2px solid var(--accent);
}
.runs-table th {
padding: 1rem;
text-align: left;
font-weight: 600;
color: var(--text-primary);
font-size: 0.875rem;
text-transform: uppercase;
letter-spacing: 0.5px;
}
.runs-table td {
padding: 0.75rem 1rem;
border-bottom: 1px solid rgba(224, 224, 224, 0.1);
color: var(--text-secondary);
font-size: 0.875rem;
}
.runs-table tbody tr:hover {
background-color: rgba(0, 180, 216, 0.05);
}
/* Status badge */
.status-badge {
display: inline-block;
padding: 0.35rem 0.75rem;
border-radius: 4px;
font-size: 0.75rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.5px;
}
.status-badge.completed {
background-color: rgba(39, 174, 96, 0.2);
color: var(--success);
}
.status-badge.in-progress {
background-color: rgba(243, 156, 18, 0.2);
color: var(--warning);
}
.status-badge.failed {
background-color: rgba(231, 76, 60, 0.2);
color: var(--danger);
}
/* Responsive */
@media (max-width: 768px) {
.page-header h1 {
font-size: 1.75rem;
}
.stats-grid {
grid-template-columns: 1fr;
}
.chart-container {
height: 300px;
}
.quick-links {
flex-direction: column;
}
.quick-link {
width: 100%;
justify-content: center;
}
.runs-table th,
.runs-table td {
padding: 0.5rem 0.75rem;
font-size: 0.75rem;
}
}
/* Empty state */
.empty-state {
text-align: center;
padding: 2rem;
color: var(--text-secondary);
}
.empty-state p {
margin: 0;
}
/* Number formatting */
.number-highlight {
color: var(--accent);
font-weight: 600;
}
.percentage-highlight {
color: var(--warning);
font-weight: 600;
}
.percentage-highlight.high {
color: var(--danger);
}
</style>
{% endblock %}
{% block content %}
<div class="page-header">
<h1>Toxicity Analysis</h1>
<p class="subtitle">
{{ stats.total_scored_posts | format_number }} / {{ stats.total_posts | format_number }} posts scored
<span style="margin: 0 0.5rem;"></span>
{{ stats.total_scored_mentions | format_number }} / {{ stats.total_mentions | format_number }} mentions scored
</p>
</div>
<!-- Stats Grid -->
<div class="stats-grid">
<!-- Total Scored Card -->
<div class="stat-card">
<div>
<div class="stat-card-label">Total Scored</div>
<div class="stat-card-value">{{ (stats.total_scored_posts + stats.total_scored_mentions) | format_number }}</div>
</div>
<div class="stat-card-detail">
{{ stats.total_scored_posts | format_number }} posts
<span style="color: var(--text-secondary); margin: 0 0.25rem;">+</span>
{{ stats.total_scored_mentions | format_number }} mentions
</div>
</div>
<!-- Flagged Posts Card -->
<div class="stat-card {% if stats.flagged_posts > 0 and (stats.flagged_posts / (stats.total_scored_posts or 1)) > 0.05 %}danger{% endif %}">
<div>
<div class="stat-card-label">Flagged Posts</div>
<div class="stat-card-value">{{ stats.flagged_posts | format_number }}</div>
</div>
<div class="stat-card-detail">
<span class="{% if stats.flagged_posts > 0 and (stats.flagged_posts / (stats.total_scored_posts or 1)) > 0.05 %}percentage-highlight high{% else %}percentage-highlight{% endif %}">
{{ "%.2f" | format(100.0 * stats.flagged_posts / (stats.total_scored_posts or 1)) }}%
</span>
of scored posts
</div>
<div class="percentage-bar">
<div class="percentage-bar-fill" style="width: {{ 100.0 * stats.flagged_posts / (stats.total_scored_posts or 1) }}%"></div>
</div>
</div>
<!-- Flagged Mentions Card -->
<div class="stat-card">
<div>
<div class="stat-card-label">Flagged Mentions</div>
<div class="stat-card-value">{{ stats.flagged_mentions | format_number }}</div>
</div>
<div class="stat-card-detail">
<span class="percentage-highlight">
{{ "%.2f" | format(100.0 * stats.flagged_mentions / (stats.total_scored_mentions or 1)) }}%
</span>
of scored mentions
</div>
<div class="percentage-bar">
<div class="percentage-bar-fill" style="width: {{ 100.0 * stats.flagged_mentions / (stats.total_scored_mentions or 1) }}%"></div>
</div>
</div>
<!-- Avg Toxicity Card -->
<div class="stat-card">
<div>
<div class="stat-card-label">Average Toxicity</div>
<div class="stat-card-value">{{ "%.1f" | format(100.0 * ((stats.avg_toxicity_posts + stats.avg_toxicity_mentions) / 2.0)) }}%</div>
</div>
<div class="stat-card-detail">
Posts: {{ "%.2f" | format(100.0 * stats.avg_toxicity_posts) }}%
<span style="color: var(--text-secondary); margin: 0 0.25rem;"></span>
Mentions: {{ "%.2f" | format(100.0 * stats.avg_toxicity_mentions) }}%
</div>
<div class="percentage-bar">
<div class="percentage-bar-fill" style="width: {{ 100.0 * ((stats.avg_toxicity_posts + stats.avg_toxicity_mentions) / 2.0) }}%"></div>
</div>
</div>
</div>
<!-- Trend Chart -->
<div class="card">
<div class="card-title">Toxicity Trends Over Time</div>
<div class="chart-container">
<canvas id="trendChart"></canvas>
</div>
</div>
<!-- Category Breakdown -->
<div class="card">
<div class="card-title">Toxicity by Category</div>
<div class="chart-container horizontal">
<canvas id="categoriesChart"></canvas>
</div>
</div>
<!-- Recent Analysis Runs -->
<div class="card">
<div class="card-title">Recent Analysis Runs</div>
{% if runs %}
<div style="overflow-x: auto;">
<table class="runs-table">
<thead>
<tr>
<th>Started</th>
<th>Duration</th>
<th>Posts Scored</th>
<th>Mentions Scored</th>
<th>Errors</th>
<th>Cost</th>
<th>Status</th>
</tr>
</thead>
<tbody>
{% for run in runs[:5] %}
<tr>
<td>{{ run.started_at | time_ago }}</td>
<td>{% if run.duration_secs is not none %}{{ "%.0f" | format(run.duration_secs | float) }}s{% else %}—{% endif %}</td>
<td>{{ run.posts_scored | format_number }}</td>
<td>{{ run.mentions_scored | format_number }}</td>
<td>{{ run.errors }}</td>
<td>${{ "%.4f" | format(run.cost_usd | default(0) | float) }}</td>
<td>
<span class="status-badge {% if run.status == 'completed' %}completed{% elif run.status == 'in_progress' %}in-progress{% elif run.status == 'failed' %}failed{% endif %}">
{{ run.status }}
</span>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
{% else %}
<div class="empty-state">
<p>No analysis runs yet. Start a new analysis to see results here.</p>
</div>
{% endif %}
</div>
<!-- Quick Links -->
<div style="margin-top: 2rem;">
<div class="quick-links">
<a href="{{ url_for('analysis.flagged') }}" class="quick-link">View Flagged Content</a>
<a href="{{ url_for('analysis.accounts') }}" class="quick-link">Account Breakdown</a>
</div>
</div>
<!-- Chart.js Script -->
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.7/dist/chart.umd.min.js"></script>
<script>
// Chart color scheme
const chartColors = {
accent: '#00b4d8',
orange: '#e67e22',
gridLine: '#2a2a3e',
text: '#e0e0e0',
categories: [
'#00b4d8', // 1
'#e67e22', // 2
'#9b59b6', // 3
'#1abc9c', // 4
'#e74c3c', // 5
'#f39c12', // 6
'#3498db', // 7
'#2ecc71' // 8
]
};
// Trend Chart
{% if trend_json %}
const trendData = {{ trend_json | safe }};
const trendCtx = document.getElementById('trendChart').getContext('2d');
const trendChart = new Chart(trendCtx, {
type: 'line',
data: {
labels: trendData.map(d => d.week),
datasets: [
{
label: 'Avg Post Toxicity',
data: trendData.map(d => d.avg_post_toxicity),
borderColor: chartColors.accent,
backgroundColor: 'rgba(0, 180, 216, 0.05)',
borderWidth: 2,
tension: 0.4,
fill: true,
yAxisID: 'y',
pointBackgroundColor: chartColors.accent,
pointBorderColor: '#1a1a2e',
pointBorderWidth: 2,
pointRadius: 5,
pointHoverRadius: 7
},
{
label: 'Avg Mention Toxicity',
data: trendData.map(d => d.avg_mention_toxicity),
borderColor: chartColors.orange,
backgroundColor: 'rgba(230, 126, 34, 0.05)',
borderWidth: 2,
tension: 0.4,
fill: true,
yAxisID: 'y',
pointBackgroundColor: chartColors.orange,
pointBorderColor: '#1a1a2e',
pointBorderWidth: 2,
pointRadius: 5,
pointHoverRadius: 7
},
{
label: 'Flagged Posts',
data: trendData.map(d => d.flagged_posts),
type: 'bar',
borderColor: 'rgba(231, 76, 60, 0.5)',
backgroundColor: 'rgba(231, 76, 60, 0.2)',
yAxisID: 'y1',
borderWidth: 1,
barThickness: 8,
categoryPercentage: 0.8,
maxBarThickness: 15
},
{
label: 'Flagged Mentions',
data: trendData.map(d => d.flagged_mentions),
type: 'bar',
borderColor: 'rgba(243, 156, 18, 0.5)',
backgroundColor: 'rgba(243, 156, 18, 0.2)',
yAxisID: 'y1',
borderWidth: 1,
barThickness: 8,
categoryPercentage: 0.8,
maxBarThickness: 15
}
]
},
options: {
responsive: true,
maintainAspectRatio: false,
interaction: {
mode: 'index',
intersect: false
},
plugins: {
legend: {
display: true,
labels: {
color: chartColors.text,
usePointStyle: true,
padding: 15,
font: {
size: 12
}
}
},
tooltip: {
backgroundColor: '#0f3460',
titleColor: chartColors.text,
bodyColor: chartColors.text,
borderColor: chartColors.accent,
borderWidth: 1,
padding: 10,
displayColors: true
}
},
scales: {
x: {
grid: {
color: chartColors.gridLine,
drawBorder: false
},
ticks: {
color: chartColors.text,
font: {
size: 11
}
}
},
y: {
type: 'linear',
display: true,
position: 'left',
min: 0,
max: 1,
ticks: {
color: chartColors.text,
font: {
size: 11
},
callback: function(value) {
return (value * 100).toFixed(0) + '%';
}
},
grid: {
color: chartColors.gridLine,
drawBorder: false
},
title: {
display: true,
text: 'Toxicity Score',
color: chartColors.text,
font: {
size: 12,
weight: 'bold'
}
}
},
y1: {
type: 'linear',
display: true,
position: 'right',
grid: {
drawOnChartArea: false
},
ticks: {
color: chartColors.text,
font: {
size: 11
}
},
title: {
display: true,
text: 'Flagged Count',
color: chartColors.text,
font: {
size: 12,
weight: 'bold'
}
}
}
}
}
});
{% endif %}
// Category Chart
{% if categories_json %}
const categoriesData = {{ categories_json | safe }};
const categoryNames = {{ categories | tojson | safe }};
// Calculate dynamic max for better readability with smart scaling
const categoryValues = categoryNames.map(cat => categoriesData[cat] || 0);
const maxCategoryValue = Math.max(...categoryValues);
// Smart scaling: round up to next nice number with 20% padding
let dynamicMax;
const paddedMax = maxCategoryValue * 1.2;
if (paddedMax <= 0.01) {
dynamicMax = 0.01; // 1%
} else if (paddedMax <= 0.02) {
dynamicMax = 0.02; // 2%
} else if (paddedMax <= 0.05) {
dynamicMax = 0.05; // 5%
} else if (paddedMax <= 0.1) {
dynamicMax = 0.1; // 10%
} else if (paddedMax <= 0.2) {
dynamicMax = 0.2; // 20%
} else if (paddedMax <= 0.5) {
dynamicMax = 0.5; // 50%
} else {
dynamicMax = 1.0; // 100%
}
const categoriesCtx = document.getElementById('categoriesChart').getContext('2d');
const categoriesChart = new Chart(categoriesCtx, {
type: 'bar',
data: {
labels: categoryNames,
datasets: [
{
label: 'Average Toxicity Score',
data: categoryValues,
backgroundColor: chartColors.categories,
borderColor: chartColors.categories.map(c => c.replace('0.', '1.')),
borderWidth: 1.5,
borderRadius: 4
}
]
},
options: {
indexAxis: 'y',
responsive: true,
maintainAspectRatio: false,
plugins: {
legend: {
display: false
},
tooltip: {
backgroundColor: '#0f3460',
titleColor: chartColors.text,
bodyColor: chartColors.text,
borderColor: chartColors.accent,
borderWidth: 1,
padding: 10,
callbacks: {
label: function(context) {
return 'Score: ' + (context.parsed.x * 100).toFixed(2) + '%';
}
}
}
},
scales: {
x: {
min: 0,
max: dynamicMax,
grid: {
color: chartColors.gridLine,
drawBorder: false
},
ticks: {
color: chartColors.text,
font: {
size: 11
},
callback: function(value) {
// Use more decimal places for small values
if (dynamicMax <= 0.01) {
return (value * 100).toFixed(2) + '%';
} else if (dynamicMax <= 0.1) {
return (value * 100).toFixed(1) + '%';
} else {
return (value * 100).toFixed(0) + '%';
}
}
},
title: {
display: true,
text: 'Average Toxicity',
color: chartColors.text,
font: {
size: 12,
weight: 'bold'
}
}
},
y: {
grid: {
drawOnChartArea: false,
drawBorder: false
},
ticks: {
color: chartColors.text,
font: {
size: 11
}
}
}
}
}
});
{% endif %}
</script>
{% endblock %}

View file

@ -242,6 +242,7 @@
<a href="{{ url_for('index') }}" class="{{ 'active' if request.endpoint == 'index' }}">Dashboard</a>
<a href="{{ url_for('accounts_list') }}" class="{{ 'active' if request.endpoint == 'accounts_list' }}">Accounts</a>
<a href="{{ url_for('statuses_list') }}" class="{{ 'active' if request.endpoint == 'statuses_list' }}">Statuses</a>
<a href="{{ url_for('analysis_dashboard') }}" class="{{ 'active' if 'analysis' in request.endpoint }}">Analysis</a>
<a href="{{ url_for('export_csv') }}">Export CSV</a>
<a href="{{ url_for('api_stats') }}">API</a>
</div>

785
app/templates/flagged.html Normal file
View file

@ -0,0 +1,785 @@
{% extends "base.html" %}
{% block title %}Flagged Content{% endblock %}
{% block content %}
<div class="flagged-container">
<!-- Page Header -->
<div class="page-header">
<h1>Flagged Content</h1>
<span class="total-badge">{{ total | format_number }}</span>
</div>
<!-- Filter Bar -->
<div class="filter-bar">
<form method="get" action="{{ url_for('analysis.flagged') }}" class="filter-form">
<div class="filter-group">
<label for="content-type">Type:</label>
<select id="content-type" name="content_type" class="filter-select">
<option value="">All Types</option>
<option value="post" {% if content_type == 'post' %}selected{% endif %}>Post</option>
<option value="reply" {% if content_type == 'reply' %}selected{% endif %}>Reply</option>
<option value="mention" {% if content_type == 'mention' %}selected{% endif %}>Mention</option>
</select>
</div>
<div class="filter-group">
<label for="category">Category:</label>
<select id="category" name="category" class="filter-select">
<option value="">All Categories</option>
{% for cat in categories %}
<option value="{{ cat }}" {% if category == cat %}selected{% endif %}>{{ cat }}</option>
{% endfor %}
</select>
</div>
<div class="filter-group">
<label for="account-did">Account:</label>
<select id="account-did" name="account_did" class="filter-select">
<option value="">All Accounts</option>
{% for acc in accounts %}
<option value="{{ acc.did }}" {% if account_did == acc.did %}selected{% endif %}>{{ acc.handle }}</option>
{% endfor %}
</select>
</div>
<div class="filter-group">
<label for="threshold">Threshold:</label>
<input type="number" id="threshold" name="threshold" min="0.0" max="1.0" step="0.1" value="{{ threshold or 0.5 }}" class="filter-input" placeholder="0.5">
</div>
<div class="filter-group">
<label for="review-status">Review Status:</label>
<select id="review-status" name="review_status" class="filter-select">
<option value="">All</option>
<option value="unreviewed" {% if review_status == 'unreviewed' %}selected{% endif %}>Unreviewed</option>
<option value="correct" {% if review_status == 'correct' %}selected{% endif %}>✓ Correct</option>
<option value="incorrect" {% if review_status == 'incorrect' %}selected{% endif %}>✗ Incorrect</option>
<option value="unsure" {% if review_status == 'unsure' %}selected{% endif %}>? Unsure</option>
</select>
</div>
<div class="filter-group">
<label for="date-from">From Date:</label>
<input type="date" id="date-from" name="date_from" value="{{ date_from or '' }}" class="filter-input">
</div>
<div class="filter-group">
<label for="date-to">To Date:</label>
<input type="date" id="date-to" name="date_to" value="{{ date_to or '' }}" class="filter-input">
</div>
<button type="submit" class="btn-apply">Apply Filters</button>
</form>
</div>
<!-- Content Table -->
{% if items %}
{% macro sort_header(col, label) %}
{% set new_dir = 'asc' if (sort == col and direction == 'desc') else 'desc' %}
<a href="{{ url_for('analysis.flagged', content_type=content_type, category=category, account_did=account_did, threshold=threshold, review_status=review_status, date_from=date_from, date_to=date_to, sort=col, dir=new_dir) }}" class="sort-link">
{{ label }}
{% if sort == col %}
<span class="sort-icon">{{ '▼' if direction == 'desc' else '▲' }}</span>
{% endif %}
</a>
{% endmacro %}
<div class="table-wrapper">
<table class="flagged-table">
<thead>
<tr>
<th>Type</th>
<th>{{ sort_header('author_handle', 'Author') }}</th>
<th>Content</th>
<th>{{ sort_header('overall', 'Score') }}</th>
<th>Category</th>
<th>{{ sort_header('created_at', 'Created') }}</th>
<th>Review</th>
</tr>
</thead>
<tbody>
{% for item in items %}
<tr class="item-row">
<!-- Type Badge -->
<td class="col-type">
<span class="badge badge-{{ item.item_type }}">
{% if item.item_type == 'post' %}
Post
{% elif item.item_type == 'reply' %}
Reply
{% elif item.item_type == 'mention' %}
Mention
{% endif %}
</span>
</td>
<!-- Author -->
<td class="col-author">
{% if item.author_handle %}
<a href="https://bsky.app/profile/{{ item.author_handle }}" target="_blank" rel="noopener" class="author-link">
@{{ item.author_handle }}
</a>
{% else %}
<span class="author-did" title="{{ item.author_did }}">{{ item.author_did[:30] }}…</span>
{% endif %}
{% if item.item_type == 'mention' and item.mentioned_handle %}
<span class="mention-arrow"></span>
<a href="https://bsky.app/profile/{{ item.mentioned_handle }}" target="_blank" rel="noopener" class="author-link">
@{{ item.mentioned_handle }}
</a>
{% endif %}
</td>
<!-- Content Text -->
<td class="col-text">
{% if item.source_type == 'post' %}
<a href="{{ url_for('statuses.detail', encoded_uri=encode_uri(item.item_id)) }}" class="content-link">
{{ item.text | truncate_text(200) }}
</a>
{% else %}
{# Convert at://did:plc:xxx/app.bsky.feed.post/yyy to https://bsky.app/profile/handle/post/yyy #}
{% set uri_parts = item.item_id.replace('at://', '').split('/') %}
{% if uri_parts|length >= 3 and item.author_handle %}
<a href="https://bsky.app/profile/{{ item.author_handle }}/post/{{ uri_parts[2] }}" target="_blank" rel="noopener" class="content-link">
{{ item.text | truncate_text(200) }}
</a>
{% else %}
<span class="content-text">{{ item.text | truncate_text(200) }}</span>
{% endif %}
{% endif %}
</td>
<!-- Score with Bar -->
<td class="col-score">
<div class="score-bar-container">
{% set score_pct = (item.overall * 100) | int %}
{% if item.overall < 0.3 %}
{% set bar_class = 'score-bar-low' %}
{% elif item.overall < 0.6 %}
{% set bar_class = 'score-bar-medium' %}
{% else %}
{% set bar_class = 'score-bar-high' %}
{% endif %}
<div class="score-bar {{ bar_class }}" style="width: {{ score_pct }}%"></div>
<span class="score-number">{{ "%.2f" | format(item.overall) }}</span>
</div>
</td>
<!-- Top Category -->
<td class="col-category">
{% if item.top_category %}
<span class="badge badge-category">{{ item.top_category }}</span>
{% else %}
<span class="text-muted"></span>
{% endif %}
</td>
<!-- Created Time -->
<td class="col-created">
<span class="time-ago" title="{{ item.created_at }}">
{{ item.created_at | time_ago }}
</span>
</td>
<!-- Review Buttons -->
<td class="col-review">
<div class="review-buttons" data-item-id="{{ item.item_id }}" data-source-type="{{ item.source_type }}" data-current-status="{{ item.review_status or '' }}">
<button class="btn-review btn-correct {% if item.review_status == 'correct' %}active{% endif %}" data-status="correct" title="Mark as correctly flagged"></button>
<button class="btn-review btn-incorrect {% if item.review_status == 'incorrect' %}active{% endif %}" data-status="incorrect" title="Mark as incorrectly flagged"></button>
<button class="btn-review btn-unsure {% if item.review_status == 'unsure' %}active{% endif %}" data-status="unsure" title="Mark as unsure">?</button>
</div>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
<!-- Pagination -->
{% if total_pages > 1 %}
<div class="pagination">
{% if page > 1 %}
<a href="{{ url_for('analysis.flagged', page=1, content_type=content_type, category=category, account_did=account_did, threshold=threshold, review_status=review_status, date_from=date_from, date_to=date_to, sort=sort, dir=direction) }}" class="btn-pagination">First</a>
<a href="{{ url_for('analysis.flagged', page=page-1, content_type=content_type, category=category, account_did=account_did, threshold=threshold, review_status=review_status, date_from=date_from, date_to=date_to, sort=sort, dir=direction) }}" class="btn-pagination">Previous</a>
{% endif %}
<span class="pagination-info">Page {{ page }} of {{ total_pages }}</span>
{% if page < total_pages %}
<a href="{{ url_for('analysis.flagged', page=page+1, content_type=content_type, category=category, account_did=account_did, threshold=threshold, review_status=review_status, date_from=date_from, date_to=date_to, sort=sort, dir=direction) }}" class="btn-pagination">Next</a>
<a href="{{ url_for('analysis.flagged', page=total_pages, content_type=content_type, category=category, account_did=account_did, threshold=threshold, review_status=review_status, date_from=date_from, date_to=date_to, sort=sort, dir=direction) }}" class="btn-pagination">Last</a>
{% endif %}
</div>
{% endif %}
{% else %}
<!-- Empty State -->
<div class="empty-state">
<p class="empty-icon"></p>
<p class="empty-text">No flagged content found</p>
<p class="empty-subtext">Try adjusting your filters or threshold</p>
</div>
{% endif %}
</div>
{% endblock %}
{% block extra_css %}
<style>
:root {
--dark-bg: #1a1a2e;
--dark-card: #16213e;
--dark-nav: #0f3460;
--dark-text: #e0e0e0;
--accent-primary: #00b4d8;
--badge-post: #00b4d8;
--badge-reply: #9b59b6;
--badge-mention: #2ecc71;
--tox-low: #2ecc71;
--tox-medium: #f39c12;
--tox-high: #e74c3c;
}
.flagged-container {
padding: 2rem;
max-width: 1400px;
margin: 0 auto;
}
/* Page Header */
.page-header {
display: flex;
align-items: center;
gap: 1rem;
margin-bottom: 2rem;
}
.page-header h1 {
font-size: 2rem;
font-weight: 700;
color: var(--dark-text);
margin: 0;
}
.total-badge {
background: var(--accent-primary);
color: var(--dark-bg);
font-weight: 600;
padding: 0.5rem 1rem;
border-radius: 2rem;
font-size: 0.9rem;
}
/* Filter Bar */
.filter-bar {
background: var(--dark-card);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 0.5rem;
padding: 1.5rem;
margin-bottom: 2rem;
}
.filter-form {
display: flex;
flex-wrap: wrap;
gap: 1rem;
align-items: flex-end;
}
.filter-group {
display: flex;
flex-direction: column;
gap: 0.5rem;
flex: 1;
min-width: 150px;
}
.filter-group label {
font-size: 0.9rem;
font-weight: 600;
color: var(--dark-text);
}
.filter-select,
.filter-input {
background: var(--dark-bg);
border: 1px solid rgba(255, 255, 255, 0.2);
border-radius: 0.375rem;
color: var(--dark-text);
padding: 0.625rem;
font-size: 0.9rem;
font-family: inherit;
}
.filter-select:hover,
.filter-input:hover {
border-color: rgba(255, 255, 255, 0.3);
}
.filter-select:focus,
.filter-input:focus {
outline: none;
border-color: var(--accent-primary);
background: var(--dark-bg);
color: var(--dark-text);
}
.btn-apply {
background: var(--accent-primary);
color: var(--dark-bg);
border: none;
border-radius: 0.375rem;
padding: 0.625rem 1.25rem;
font-weight: 600;
cursor: pointer;
font-size: 0.9rem;
transition: all 0.2s ease;
}
.btn-apply:hover {
opacity: 0.9;
transform: translateY(-1px);
}
.btn-apply:active {
transform: translateY(0);
}
/* Table Wrapper */
.table-wrapper {
background: var(--dark-card);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 0.5rem;
overflow-x: auto;
margin-bottom: 2rem;
}
/* Table Styles */
.flagged-table {
width: 100%;
border-collapse: collapse;
font-size: 0.9rem;
}
.flagged-table thead {
background: rgba(0, 0, 0, 0.3);
border-bottom: 2px solid rgba(255, 255, 255, 0.1);
}
.flagged-table th {
padding: 1rem;
text-align: left;
font-weight: 600;
color: var(--dark-text);
white-space: nowrap;
}
/* Sort Links */
.sort-link {
color: var(--dark-text);
text-decoration: none;
display: inline-flex;
align-items: center;
gap: 0.25rem;
transition: color 0.2s ease;
}
.sort-link:hover {
color: var(--accent-primary);
}
.sort-icon {
font-size: 0.8rem;
opacity: 0.7;
}
.flagged-table td {
padding: 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.05);
color: var(--dark-text);
}
.flagged-table tbody tr:hover {
background: rgba(0, 180, 216, 0.05);
}
/* Column Styles */
.col-type {
width: 90px;
}
.col-author {
width: 200px;
}
.col-text {
min-width: 300px;
}
.col-score {
width: 150px;
}
.col-category {
width: 140px;
}
.col-created {
width: 120px;
}
.col-review {
width: 130px;
}
/* Badges */
.badge {
display: inline-block;
padding: 0.35rem 0.75rem;
border-radius: 0.25rem;
font-size: 0.8rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.5px;
}
.badge-post {
background: rgba(0, 180, 216, 0.2);
color: var(--badge-post);
}
.badge-reply {
background: rgba(155, 89, 182, 0.2);
color: var(--badge-reply);
}
.badge-mention {
background: rgba(46, 204, 113, 0.2);
color: var(--badge-mention);
}
.badge-category {
background: rgba(0, 180, 216, 0.15);
color: var(--accent-primary);
}
/* Author Links */
.author-link {
color: var(--accent-primary);
text-decoration: none;
transition: color 0.2s ease;
}
.author-link:hover {
color: rgba(0, 180, 216, 0.8);
text-decoration: underline;
}
.mention-arrow {
color: rgba(255, 255, 255, 0.3);
margin: 0 0.5rem;
}
/* Content Link */
.content-link {
color: var(--dark-text);
text-decoration: none;
transition: color 0.2s ease;
}
.content-link:hover {
color: var(--accent-primary);
}
.content-text {
color: rgba(255, 255, 255, 0.7);
}
/* Score Bar */
.score-bar-container {
position: relative;
height: 30px;
background: rgba(255, 255, 255, 0.05);
border-radius: 0.25rem;
overflow: hidden;
display: flex;
align-items: center;
padding: 0 0.5rem;
}
.score-bar {
position: absolute;
height: 100%;
left: 0;
top: 0;
transition: width 0.3s ease;
}
.score-bar-low {
background: linear-gradient(90deg, rgba(46, 204, 113, 0.3), rgba(46, 204, 113, 0.5));
}
.score-bar-medium {
background: linear-gradient(90deg, rgba(243, 156, 18, 0.3), rgba(243, 156, 18, 0.5));
}
.score-bar-high {
background: linear-gradient(90deg, rgba(231, 76, 60, 0.3), rgba(231, 76, 60, 0.5));
}
.score-number {
position: relative;
z-index: 1;
font-weight: 600;
color: var(--dark-text);
font-size: 0.85rem;
}
/* Time Ago */
.time-ago {
color: rgba(255, 255, 255, 0.6);
font-size: 0.85rem;
cursor: help;
}
.time-ago:hover {
color: var(--dark-text);
}
.text-muted {
color: rgba(255, 255, 255, 0.3);
}
/* Review Buttons */
.review-buttons {
display: flex;
gap: 0.25rem;
justify-content: center;
}
.btn-review {
background: var(--dark-bg);
border: 1px solid rgba(255, 255, 255, 0.2);
color: var(--dark-text);
width: 32px;
height: 32px;
border-radius: 0.25rem;
font-size: 1rem;
cursor: pointer;
transition: all 0.2s ease;
display: flex;
align-items: center;
justify-content: center;
}
.btn-review:hover {
transform: translateY(-2px);
border-color: rgba(255, 255, 255, 0.4);
}
.btn-correct:hover, .btn-correct.active {
background: var(--tox-low);
border-color: var(--tox-low);
color: var(--dark-bg);
}
.btn-incorrect:hover, .btn-incorrect.active {
background: var(--tox-high);
border-color: var(--tox-high);
color: var(--dark-bg);
}
.btn-unsure:hover, .btn-unsure.active {
background: var(--tox-medium);
border-color: var(--tox-medium);
color: var(--dark-bg);
}
.btn-review.active {
font-weight: 700;
transform: none;
}
.btn-review.active:hover {
opacity: 0.8;
}
/* Empty State */
.empty-state {
text-align: center;
padding: 4rem 2rem;
background: var(--dark-card);
border: 2px dashed rgba(255, 255, 255, 0.2);
border-radius: 0.5rem;
}
.empty-icon {
font-size: 3rem;
color: rgba(255, 255, 255, 0.2);
margin: 0 0 1rem 0;
}
.empty-text {
font-size: 1.2rem;
font-weight: 600;
color: var(--dark-text);
margin: 0 0 0.5rem 0;
}
.empty-subtext {
color: rgba(255, 255, 255, 0.5);
margin: 0;
}
/* Pagination */
.pagination {
display: flex;
justify-content: center;
align-items: center;
gap: 1rem;
margin-top: 2rem;
}
.pagination-info {
color: var(--dark-text);
font-weight: 500;
min-width: 150px;
text-align: center;
}
.btn-pagination {
background: var(--dark-card);
color: var(--accent-primary);
border: 1px solid var(--accent-primary);
padding: 0.5rem 1rem;
border-radius: 0.375rem;
text-decoration: none;
font-weight: 600;
font-size: 0.9rem;
transition: all 0.2s ease;
cursor: pointer;
}
.btn-pagination:hover {
background: var(--accent-primary);
color: var(--dark-bg);
}
.btn-pagination:active {
transform: scale(0.98);
}
/* Responsive */
@media (max-width: 1024px) {
.filter-form {
flex-direction: column;
}
.filter-group {
width: 100%;
}
.col-text {
min-width: 250px;
}
}
@media (max-width: 768px) {
.flagged-container {
padding: 1rem;
}
.page-header {
flex-direction: column;
align-items: flex-start;
}
.page-header h1 {
font-size: 1.5rem;
}
.table-wrapper {
font-size: 0.8rem;
}
.flagged-table th,
.flagged-table td {
padding: 0.75rem 0.5rem;
}
.col-author,
.col-text {
min-width: 180px;
}
.col-created {
width: 100px;
}
}
</style>
{% endblock %}
{% block extra_js %}
<script>
document.addEventListener('DOMContentLoaded', function() {
// Handle review button clicks
document.querySelectorAll('.btn-review').forEach(button => {
button.addEventListener('click', async function() {
const buttonContainer = this.parentElement;
const itemId = buttonContainer.dataset.itemId;
const sourceType = buttonContainer.dataset.sourceType;
const currentStatus = buttonContainer.dataset.currentStatus;
const clickedStatus = this.dataset.status;
// If clicking the same status, do nothing (it's already set)
if (currentStatus === clickedStatus) {
return;
}
// Disable all buttons while processing
buttonContainer.querySelectorAll('.btn-review').forEach(btn => btn.disabled = true);
try {
const response = await fetch('/api/review/submit', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
item_id: itemId,
source_type: sourceType,
review_status: clickedStatus,
}),
});
if (response.ok) {
// Update current status
buttonContainer.dataset.currentStatus = clickedStatus;
// Remove active class from all buttons
buttonContainer.querySelectorAll('.btn-review').forEach(btn => {
btn.classList.remove('active');
});
// Add active class to clicked button
this.classList.add('active');
// Re-enable all buttons
buttonContainer.querySelectorAll('.btn-review').forEach(btn => btn.disabled = false);
} else {
// Re-enable buttons on error
buttonContainer.querySelectorAll('.btn-review').forEach(btn => btn.disabled = false);
alert('Failed to submit review. Please try again.');
}
} catch (error) {
// Re-enable buttons on error
buttonContainer.querySelectorAll('.btn-review').forEach(btn => btn.disabled = false);
console.error('Error submitting review:', error);
alert('Failed to submit review. Please try again.');
}
});
});
});
</script>
{% endblock %}

View file

@ -23,6 +23,58 @@ logger = logging.getLogger(__name__)
app = Flask(__name__)
app.secret_key = os.environ.get("FLASK_SECRET_KEY", "dev-secret-key")
# Template filters
@app.template_filter('format_number')
def format_number(value):
"""Format number with commas."""
try:
return f"{int(value):,}"
except (ValueError, TypeError):
return value
@app.template_filter('time_ago')
def time_ago(dt):
"""Convert datetime to time ago string."""
if not dt:
return "Never"
from datetime import datetime, timezone
now = datetime.now(timezone.utc)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
diff = now - dt
seconds = diff.total_seconds()
if seconds < 60:
return "just now"
elif seconds < 3600:
mins = int(seconds / 60)
return f"{mins}m ago"
elif seconds < 86400:
hours = int(seconds / 3600)
return f"{hours}h ago"
elif seconds < 604800:
days = int(seconds / 86400)
return f"{days}d ago"
else:
weeks = int(seconds / 604800)
return f"{weeks}w ago"
@app.template_filter('truncate_text')
def truncate_text(text, length=200):
"""Truncate text to specified length."""
if not text:
return ""
from bs4 import BeautifulSoup
# Strip HTML tags first
text = BeautifulSoup(text, 'html.parser').get_text()
if len(text) <= length:
return text
return text[:length] + "..."
# Initialize database on startup
with app.app_context():
init_db()

View file

@ -40,6 +40,7 @@ services:
environment:
DATABASE_URL: postgresql://collector:${POSTGRES_PASSWORD:-collector_secret}@db:5432/mastodon_collector
POLL_INTERVAL_SECONDS: ${POLL_INTERVAL_SECONDS:-14400}
OPENAI_API_KEY: ${OPENAI_API_KEY}
volumes:
- ./accounts.txt:/app/accounts.txt
depends_on: