subcultureofone/tkr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor-storage (#19)

Browse Source 
Move ticks from filesystem into database.

I'm going to preserve the filesystem and just delete it manually. I'll add a login warning, but I'm pretty sure I'm the only person who'll ever be affected by this.

Co-authored-by: Greg Sarjeant <1686767+gsarjeant@users.noreply.github.com>
Reviewed-on: https://gitea.subcultureofone.org/greg/tkr/pulls/19
This commit is contained in:
greg 2025-07-24 02:12:31 +00:00
parent 7b7f8d205d
commit 9338332536
15 changed files with 150 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
config/migrations/000_init.sql Normal file
View File

@ -0,0 +1,31 @@
CREATE TABLE IF NOT EXISTS user (
id INTEGER PRIMARY KEY,
username TEXT NOT NULL,
display_name TEXT NOT NULL,
password_hash TEXT NULL,
about TEXT NULL,
website TEXT NULL,
mood TEXT NULL
);
CREATE TABLE IF NOT EXISTS settings (
id INTEGER PRIMARY KEY,
site_title TEXT NOT NULL,
site_description TEXT NULL,
base_url TEXT NOT NULL,
base_path TEXT NOT NULL,
items_per_page INTEGER NOT NULL,
css_id INTEGER NULL
);
CREATE TABLE IF NOT EXISTS css (
id INTEGER PRIMARY KEY,
filename TEXT UNIQUE NOT NULL,
description TEXT NULL
);
CREATE TABLE IF NOT EXISTS emoji(
id INTEGER PRIMARY KEY,
emoji TEXT UNIQUE NOT NULL,
description TEXT NOT NULL
);

5
config/migrations/003_add_tick_table.sql Normal file
View File

@ -0,0 +1,5 @@
CREATE TABLE IF NOT EXISTS tick (
id INTEGER PRIMARY KEY,
timestamp TEXT NOT NULL,
tick TEXT NOT NULL
);

1
config/migrations/004_add_tick_timestamp_index.sql Normal file
View File

@ -0,0 +1 @@
CREATE INDEX idx_tick_timestamp ON tick (timestamp DESC);

12
src/Controller/FeedController/FeedController.php
View File

@ -1,7 +1,5 @@
<?php
class FeedController extends Controller {
private ConfigModel $config;
private array $ticks;
private array $vars;
protected function render(string $templateFile, array $vars = []) {
@ -16,11 +14,13 @@ class FeedController extends Controller {
}
public function __construct(){
$this->config = ConfigModel::load();
$this->ticks = iterator_to_array(TickModel::streamTicks($this->config->itemsPerPage));
$config = ConfigModel::load();
$tickModel = new TickModel();
$ticks = iterator_to_array($tickModel->stream($config->itemsPerPage));
$this->vars = [
'config' => $this->config,
'ticks' => $this->ticks,
'config' => $config,
'ticks' => $ticks,
];
}

6
src/Controller/HomeController/HomeController.php
View File

@ -7,9 +7,10 @@ class HomeController extends Controller {
global $config;
global $user;
$tickModel = new TickModel();
$limit = $config->itemsPerPage;
$offset = ($page - 1) * $limit;
$ticks = iterator_to_array(TickModel::streamTicks($limit, $offset));
$ticks = iterator_to_array($tickModel->stream($limit, $offset));
$view = new HomeView();
$tickList = $view->renderTicksSection($config->siteDescription, $ticks, $page, $limit);
@ -29,7 +30,8 @@ class HomeController extends Controller {
if ($_SERVER['REQUEST_METHOD'] === 'POST' and isset($_POST['new_tick'])) {
// save the tick
if (trim($_POST['new_tick'])){
TickModel::save($_POST['new_tick'], $_POST['tick_mood']);
$tickModel = new TickModel();
$tickModel->insert($_POST['new_tick']);
}
}

10
src/Controller/TickController/TickController.php
View File

@ -1,10 +1,10 @@
<?php
class TickController extends Controller{
// every tick is identified by its timestamp
public function index(string $year, string $month, string $day, string $hour, string $minute, string $second){
$model = new TickModel();
$tick = $model->get($year, $month, $day, $hour, $minute, $second);
$this->render('tick.php', $tick);
//public function index(string $year, string $month, string $day, string $hour, string $minute, string $second){
public function index(int $id){
$tickModel = new TickModel();
$vars = $tickModel->get($id);
$this->render('tick.php', $vars);
}
}

6
src/Framework/Database/Database.php
View File

@ -27,10 +27,12 @@ class Database{
// The database version will just be an int
// stored as PRAGMA user_version. It will
// correspond to the most recent migration file applied to the db.
//
// I'm starting from 0, so if the user_version is NULL, I'll return -1.
private function getVersion(): int {
$db = self::get();
return $db->query("PRAGMA user_version")->fetchColumn() ?? 0;
return $db->query("PRAGMA user_version")->fetchColumn() ?? -1;
}
private function migrationNumberFromFile(string $filename): int {
@ -77,7 +79,7 @@ class Database{
return;
}
$db = get_db();
$db = self::get();
$db->beginTransaction();
try {

62
src/Framework/Filesystem/Filesystem.php
View File

@ -6,6 +6,7 @@ class Filesystem {
$this->validateStorageDir();
$this->validateStorageSubdirs();
$this->migrateTickFiles();
$this->moveTicksToDatabase();
}
// Make sure the storage/ directory exists and is writable
@ -74,6 +75,8 @@ class Filesystem {
$lines = file($filepath, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
$modified = false;
// &$line creates a reference to the line in the array ($lines)
// so I can modify it in place
foreach ($lines as &$line) {
$fields = explode('|', $line);
if (count($fields) === 2) {
@ -82,10 +85,69 @@ class Filesystem {
$modified = true;
}
}
unset($line);
if ($modified) {
file_put_contents($filepath, implode("\n", $lines) . "\n");
// TODO: log properly
}
}
// TODO: Delete this sometime before 1.0
// Move ticks into database
private function moveTicksToDatabase(){
// It's a temporary migration function, so I'm not going to sweat the
// order of operations to let me reuse the global database.
$db = Database::get();
$count = $db->query("SELECT COUNT(*) FROM tick")->fetchColumn();
// Only migrate from filesystem if there are no ticks already in the database.
if ($count !== 0){
return;
}
$files = new RecursiveIteratorIterator(
new RecursiveDirectoryIterator(TICKS_DIR, RecursiveDirectoryIterator::SKIP_DOTS)
);
foreach ($files as $file) {
if ($file->isFile() && str_ends_with($file->getFilename(), '.txt')) {
// Construct the date from the path and filename
$dir = pathinfo($file, PATHINFO_DIRNAME);
$dir_parts = explode('/', trim($dir, '/'));
[$year, $month] = array_slice($dir_parts, -2);
$day = pathinfo($file, PATHINFO_FILENAME);
$lines = file($file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
foreach ($lines as $line) {
// Get the time and the text, but discard the mood.
// I've decided against using it
$fields = explode('|', $line);
$time = $fields[0];
$tick = $fields[2];
$dateTime = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', "$year-$month-$day $time");
$tickDateTimeUTC = $dateTime->format('Y-m-d H:i:s');
$ticks[] = [$tickDateTimeUTC, $tick];
}
}
}
// Sort the ticks by dateTime
usort($ticks, function($a, $b) {
return strcmp($a[0], $b[0]);
});
// Save ticks to database
foreach ($ticks as $tick){
// Yes, silly, but I'm testing out the datetime/string SQLite conversion
$dateTime = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', "$tick[0]");
$timestamp = $dateTime->format('Y-m-d H:i:s');
$tickText = $tick[1];
$stmt = $db->prepare("INSERT INTO tick(timestamp, tick) values (?, ?)");
$stmt->execute([$timestamp, $tickText]);
}
}
}

2
src/Framework/Router/Router.php
View File

@ -21,7 +21,7 @@ class Router {
['mood', 'MoodController@handlePost', ['POST']],
['setup', 'AdminController@showSetup'],
['setup', 'AdminController@handleSetup', ['POST']],
['tick/{y}/{m}/{d}/{h}/{i}/{s}', 'TickController'],
['tick/{id}', 'TickController'],
['css/custom/{filename}.css', 'CssController@serveCustomCss'],
];

118
src/Model/TickModel/TickModel.php
View File

@ -1,105 +1,41 @@
<?php
class TickModel {
// Everything in this class just reads from and writes to the filesystem
// It doesn't maintain state, so everything's just a static function
public static function streamTicks(int $limit, int $offset = 0): Generator {
$tick_files = glob(TICKS_DIR . '/*/*/*.txt');
usort($tick_files, fn($a, $b) => strcmp($b, $a)); // sort filenames in reverse chronological order
public function stream(int $limit, int $offset = 0): Generator {
global $db;
$count = 0;
foreach ($tick_files as $file) {
// read all the ticks from the current file and reverse the order
// so the most recent ones are first
//
// each file is a single day, so we never hold more than
// one day's ticks in memory
$lines = array_reverse(
file($file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES)
);
$stmt = $db->prepare("SELECT id, timestamp, tick FROM tick ORDER BY timestamp DESC LIMIT ? OFFSET ?");
$stmt->execute([$limit, $offset]);
// split the path to the current file into the date components
$pathParts = explode('/', str_replace('\\', '/', $file));
// assign the different components to the appropriate part of the date
$year = $pathParts[count($pathParts) - 3];
$month = $pathParts[count($pathParts) - 2];
$day = pathinfo($pathParts[count($pathParts) - 1], PATHINFO_FILENAME);
foreach ($lines as $line) {
// just keep skipping ticks until we get to the starting point
if ($offset > 0) {
$offset--;
continue;
}
// Ticks are pipe-delimited: timestamp|text
// But just in case a tick contains a pipe, only split on the first one that occurs
list($time, $mood, $tick) = explode('|', $line, 3);
// Build the timestamp from the date and time
// Ticks are always stored in UTC
$timestampUTC = "$year-$month-$day $time";
yield [
'timestamp' => $timestampUTC,
'mood' => $mood,
'tick' => $tick,
];
if (++$count >= $limit) {
return;
}
}
while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
yield [
'id' => $row['id'],
'timestamp' => $row['timestamp'],
'tick' => $row['tick'],
];
}
}
public static function save(string $tick, string $mood=''): void {
// build the tick path and filename from the current time
$now = new DateTime('now', new DateTimeZone('UTC'));
public function insert(string $tick, ?DateTimeImmutable $datetime = null): void {
global $db;
$datetime ??= new DateTimeImmutable('now', new DateTimeZone('UTC'));
$timestamp = $datetime->format('Y-m-d H:i:s');
$year = $now->format('Y');
$month = $now->format('m');
$day = $now->format('d');
$time = $now->format('H:i:s');
// build the full path to the tick file
$dir = TICKS_DIR . "/$year/$month";
$filename = "$dir/$day.txt";
// create the directory if it doesn't exist
if (!is_dir($dir)) {
mkdir($dir, 0770, true);
}
// write the tick to the file (the file will be created if it doesn't exist)
$content = $time . '|' . $mood . '|' . $tick . "\n";
file_put_contents($filename, $content, FILE_APPEND);
$stmt = $db->prepare("INSERT INTO tick(timestamp, tick) values (?, ?)");
$stmt->execute([$timestamp, $tick]);
}
public static function get(string $y, string $m, string $d, string $H, string $i, string $s): array{
$tickTime = new DateTime("$y-$m-$d $H:$i:$s");
$timestamp = "$H:$i:$s";
$file = TICKS_DIR . "/$y/$m/$d.txt";
public function get(int $id): array {
global $db;
if (!file_exists($file)) {
http_response_code(404);
echo "Tick not found: $file.";
exit;
}
$stmt = $db->prepare("SELECT timestamp, tick FROM tick WHERE id=?");
$stmt->execute([$id]);
$row = $stmt->fetch(PDO::FETCH_ASSOC);
$lines = file($file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
foreach ($lines as $line) {
if (str_starts_with($line, $timestamp)) {
echo $line;
exit;
list($time, $emoji, $tick) = explode('|', $line, 3);
return [
'tickTime' => $tickTime,
'emoji' => $emoji,
'tick' => $tick,
'config' => ConfigModel::load(),
];
}
}
// TODO: Test for existence of row and handle absence.
return [
'tickTime' => $row['timestamp'],
'tick' => $row['tick'],
'config' => ConfigModel::load(),
];
}
}

3
src/View/HomeView/HomeView.php
View File

@ -13,9 +13,6 @@ class HomeView {
?>
<li class="tick" tabindex="0">
<time datetime="<?php echo $datetime->format('c') ?>"><?php echo Util::escape_html($relativeTime) ?></time>
<?php if ($config->showTickMood): ?>
<span><?php echo $tick['mood'] ?></span>
<?php endif; ?>
<span class="tick-text"><?php echo Util::linkify(Util::escape_html($tick['tick'])) ?></span>
</li>
<?php endforeach; ?>

BIN
storage/db/tkr.sqlite.bak Executable file
View File

Binary file not shown.

11
templates/feed/atom.php
View File

@ -22,17 +22,8 @@ echo '<?xml version="1.0" encoding="utf-8"?>' . "\n";
<name><?= Util::escape_xml($config->siteTitle) ?></name>
</author>
<?php foreach ($ticks as $tick):
// decompose the tick timestamp into the date/time parts
[$date, $time] = explode(' ', $tick['timestamp']);
$dateParts = explode('-', $date);
[$year, $month, $day] = $dateParts;
$timeParts = explode(':', $time);
[$hour, $minute, $second] = $timeParts;
// build the tick entry components
$tickPath = "tick/$year/$month/$day/$hour/$minute/$second";
$tickPath = "tick/" . $tick['id'];
$tickUrl = Util::escape_xml($siteUrl . $basePath . $tickPath);
$tickTime = date(DATE_ATOM, strtotime($tick['timestamp']));
$tickTitle = Util::escape_xml($tick['tick']);

12
templates/feed/rss.php
View File

@ -19,17 +19,9 @@ echo '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
<language>en-us</language>
<lastBuildDate><?php echo date(DATE_RSS); ?></lastBuildDate>
<?php foreach ($ticks as $tick):
// decompose the tick timestamp into the date/time parts
[$date, $time] = explode(' ', $tick['timestamp']);
$dateParts = explode('-', $date);
[$year, $month, $day] = $dateParts;
$timeParts = explode(':', $time);
[$hour, $minute, $second] = $timeParts;
// build the tick entry components
$tickPath = "tick/$year/$month/$day/$hour/$minute/$second";
//$tickPath = "tick/$year/$month/$day/$hour/$minute/$second";
$tickPath = "tick/" . $tick['id'];
$tickUrl = Util::escape_xml($config->baseUrl . $config->basePath . $tickPath);
$tickDate = date(DATE_RSS, strtotime($tick['timestamp']));
$tickTitle = Util::escape_xml($tick['tick']);

3
templates/partials/tick.php
View File

@ -1,5 +1,4 @@
<?php /** @var ConfigModel $config */ ?>
<?php /** @var Date $tickTime */ ?>
<?php /** @var string $tick */ ?>
<h1>Tick from <?= $tickTime->format('Y-m-d H:i:s'); ?></h1>
<h1>Tick from <?= $tickTime; ?></h1>
<p><?= Util::linkify(Util::escape_html($tick)) ?></p>