Remove backup database.

Move ticks from filesystem into database.

I'm going to preserve the filesystem and just delete it manually. I'll add a login warning, but I'm pretty sure I'm the only person who'll ever be affected by this.

Co-authored-by: Greg Sarjeant <1686767+gsarjeant@users.noreply.github.com>
Reviewed-on: https://gitea.subcultureofone.org/greg/tkr/pulls/19

.gitea/workflows/build_and_publish.yaml

@@ -1,26 +1,27 @@
-name: Run unit tests
-run-name: ${{ gitea.repository }} PHP unit tests
+name: Build and publish artifacts
+run-name: ${{ gitea.repository }} build and publish
 on:
   push:
     tags:
       - v*.**.**
 
 jobs:
-  build_and_publish:
+  build-and-publish:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: Build archive
         run: |
           tar \
-            --exclude='./storage/db' \
-            --exclude='./storage/ticks' \
-            --exclude='./storage/upload' \
-            -czpvf tkr.${{ gitea.ref_name }}.tgz \
+            --transform 's,^,tkr/,' \
+            --exclude='storage/db' \
+            --exclude='storage/ticks' \
+            --exclude='storage/upload' \
+            -czvf tkr.${{ gitea.ref_name }}.tgz \
             config public src storage templates
       - name: Push to Generic gitea registry
         run: |
-          curl
-          --user ${{ secrets.CONTAINER_REGISTRY_USERNAME}}:${{ secrets.CONTAINER_REGISTRY_TOKEN }} \
-          --upload-file tkr.${{ gitea.ref_name}}.tgz \
-          https://gitea.subcultureofone.org/api/packages/${{ secrets.CONTAINER_REGISTRY_USERNAME }}/generic/tkr/${{ gitea.ref_name }}/tkr.${{ gitea.ref_name }}.tgz
+          curl \
+            --user ${{ secrets.CONTAINER_REGISTRY_USERNAME}}:${{ secrets.CONTAINER_REGISTRY_TOKEN }} \
+            --upload-file tkr.${{ gitea.ref_name}}.tgz \
+            https://gitea.subcultureofone.org/api/packages/${{ secrets.CONTAINER_REGISTRY_USERNAME }}/generic/tkr/${{ gitea.ref_name }}/tkr.${{ gitea.ref_name }}.tgz

README.md

@@ -1,4 +1,5 @@
 # tkr
+![Unit tests status](https://gitea.subcultureofone.org/greg/tkr/actions/workflows/unit_tests.yaml/badge.svg)
 
 A lightweight, HTML-only status feed for self-hosted personal websites. Written in PHP. Heavily inspired by [status.cafe](https://status.cafe).
 
@@ -44,8 +45,8 @@ I'm trying to make sure that the HTML is both semantically valid and accessible,
 
 ## Installation
 
-1. Download the latest tkr archive from https://subcultureofone.org/files/tkr/tkr.0.7.0.zip
-1. Copy the .zip file to your server and extract it
+1. Download the latest tkr archive from [the packages page](https://gitea.subcultureofone.org/greg/tkr/packages)
+1. Copy the `.tgz` file to your server and extract it
 1. Copy the `tkr` directory to the location you want to serve it from
     * on debian-based systems, `/var/www/tkr` is recommended
 1. Make the `storage` directory writable by the web server account.

config/migrations/000_init.sql

@@ -0,0 +1,31 @@
+CREATE TABLE IF NOT EXISTS user (
+    id INTEGER PRIMARY KEY,
+    username TEXT NOT NULL,
+    display_name TEXT NOT NULL,
+    password_hash TEXT NULL,
+    about TEXT NULL,
+    website TEXT NULL,
+    mood TEXT NULL
+);
+
+CREATE TABLE IF NOT EXISTS settings (
+    id INTEGER PRIMARY KEY,
+    site_title TEXT NOT NULL,
+    site_description TEXT NULL,
+    base_url TEXT NOT NULL,
+    base_path TEXT NOT NULL,
+    items_per_page INTEGER NOT NULL,
+    css_id INTEGER NULL
+);
+
+CREATE TABLE IF NOT EXISTS css (
+    id INTEGER PRIMARY KEY,
+    filename TEXT UNIQUE NOT NULL,
+    description TEXT NULL
+);
+
+CREATE TABLE IF NOT EXISTS emoji(
+    id INTEGER PRIMARY KEY,
+    emoji TEXT UNIQUE NOT NULL,
+    description TEXT NOT NULL
+);

config/migrations/003_add_tick_table.sql

@@ -0,0 +1,5 @@
+CREATE TABLE IF NOT EXISTS tick (
+    id INTEGER PRIMARY KEY,
+    timestamp TEXT NOT NULL,
+    tick TEXT NOT NULL
+);

config/migrations/004_add_tick_timestamp_index.sql

@@ -0,0 +1 @@
+CREATE INDEX idx_tick_timestamp ON tick (timestamp DESC);

src/Controller/FeedController/FeedController.php

@@ -1,7 +1,5 @@
 <?php
 class FeedController extends Controller {
-    private ConfigModel $config;
-    private array $ticks;
     private array $vars;
 
     protected function render(string $templateFile, array $vars = []) {
@@ -16,11 +14,13 @@ class FeedController extends Controller {
     }
 
     public function __construct(){
-        $this->config = ConfigModel::load();
-        $this->ticks = iterator_to_array(TickModel::streamTicks($this->config->itemsPerPage));
+        $config = ConfigModel::load();
+        $tickModel = new TickModel();
+        $ticks = iterator_to_array($tickModel->stream($config->itemsPerPage));
+
         $this->vars = [
-            'config' => $this->config,
-            'ticks' => $this->ticks,
+            'config' => $config,
+            'ticks' => $ticks,
         ];
     }
 

src/Controller/HomeController/HomeController.php

@@ -7,9 +7,10 @@ class HomeController extends Controller {
         global $config;
         global $user;
 
+        $tickModel = new TickModel();
         $limit = $config->itemsPerPage;
         $offset = ($page - 1) * $limit;
-        $ticks = iterator_to_array(TickModel::streamTicks($limit, $offset));
+        $ticks = iterator_to_array($tickModel->stream($limit, $offset));
 
         $view = new HomeView();
         $tickList = $view->renderTicksSection($config->siteDescription, $ticks, $page, $limit);
@@ -29,7 +30,8 @@ class HomeController extends Controller {
         if ($_SERVER['REQUEST_METHOD'] === 'POST' and isset($_POST['new_tick'])) {
             // save the tick
             if (trim($_POST['new_tick'])){
-                TickModel::save($_POST['new_tick'], $_POST['tick_mood']);
+                $tickModel = new TickModel();
+                $tickModel->insert($_POST['new_tick']);
             }
         }
 

src/Controller/TickController/TickController.php

@@ -1,10 +1,10 @@
 <?php
 
 class TickController extends Controller{
-    // every tick is identified by its timestamp
-    public function index(string $year, string $month, string $day, string $hour, string $minute, string $second){
-        $model = new TickModel();
-        $tick = $model->get($year, $month, $day, $hour, $minute, $second);
-        $this->render('tick.php', $tick);
+    //public function index(string $year, string $month, string $day, string $hour, string $minute, string $second){
+    public function index(int $id){
+        $tickModel = new TickModel();
+        $vars = $tickModel->get($id);
+        $this->render('tick.php', $vars);
     }
 }

src/Framework/Database/Database.php

@@ -27,10 +27,12 @@ class Database{
     // The database version will just be an int
     // stored as PRAGMA user_version. It will
     // correspond to the most recent migration file applied to the db.
+    //
+    // I'm starting from 0, so if the user_version is NULL, I'll return -1.
     private function getVersion(): int {
         $db = self::get();
 
-        return $db->query("PRAGMA user_version")->fetchColumn() ?? 0;
+        return $db->query("PRAGMA user_version")->fetchColumn() ?? -1;
     }
 
     private function migrationNumberFromFile(string $filename): int {
@@ -77,7 +79,7 @@ class Database{
             return;
         }
 
-        $db = get_db();
+        $db = self::get();
         $db->beginTransaction();
 
         try {

src/Framework/Filesystem/Filesystem.php

@@ -6,6 +6,7 @@ class Filesystem {
         $this->validateStorageDir();
         $this->validateStorageSubdirs();
         $this->migrateTickFiles();
+        $this->moveTicksToDatabase();
     }
 
     // Make sure the storage/ directory exists and is writable
@@ -74,6 +75,8 @@ class Filesystem {
         $lines = file($filepath, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
         $modified = false;
 
+        // &$line creates a reference to the line in the array ($lines)
+        // so I can modify it in place
         foreach ($lines as &$line) {
             $fields = explode('|', $line);
             if (count($fields) === 2) {
@@ -82,10 +85,69 @@ class Filesystem {
                 $modified = true;
             }
         }
+        unset($line);
 
         if ($modified) {
             file_put_contents($filepath, implode("\n", $lines) . "\n");
             // TODO: log properly
         }
     }
+
+    // TODO: Delete this sometime before 1.0
+    // Move ticks into database
+    private function moveTicksToDatabase(){
+        // It's a temporary migration function, so I'm not going to sweat the
+        // order of operations to let me reuse the global database.
+        $db = Database::get();
+        $count = $db->query("SELECT COUNT(*) FROM tick")->fetchColumn();
+
+        // Only migrate from filesystem if there are no ticks already in the database.
+        if ($count !== 0){
+            return;
+        }
+
+        $files = new RecursiveIteratorIterator(
+            new RecursiveDirectoryIterator(TICKS_DIR, RecursiveDirectoryIterator::SKIP_DOTS)
+        );
+
+        foreach ($files as $file) {
+            if ($file->isFile() && str_ends_with($file->getFilename(), '.txt')) {
+                // Construct the date from the path and filename
+                $dir = pathinfo($file, PATHINFO_DIRNAME);
+                $dir_parts = explode('/', trim($dir, '/'));
+                [$year, $month] = array_slice($dir_parts, -2);
+                $day = pathinfo($file, PATHINFO_FILENAME);
+
+                $lines = file($file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
+                foreach ($lines as $line) {
+                    // Get the time and the text, but discard the mood.
+                    // I've decided against using it
+                    $fields = explode('|', $line);
+                    $time = $fields[0];
+                    $tick = $fields[2];
+
+                    $dateTime = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', "$year-$month-$day $time");
+                    $tickDateTimeUTC = $dateTime->format('Y-m-d H:i:s');
+
+                    $ticks[] = [$tickDateTimeUTC, $tick];
+                }
+            }
+        }
+
+        // Sort the ticks by dateTime
+        usort($ticks, function($a, $b) {
+            return strcmp($a[0], $b[0]);
+        });
+
+        // Save ticks to database
+        foreach ($ticks as $tick){
+            // Yes, silly, but I'm testing out the datetime/string SQLite conversion
+            $dateTime = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', "$tick[0]");
+            $timestamp = $dateTime->format('Y-m-d H:i:s');
+            $tickText = $tick[1];
+
+            $stmt = $db->prepare("INSERT INTO tick(timestamp, tick) values (?, ?)");
+            $stmt->execute([$timestamp, $tickText]);
+        }
+    }
 }

src/Framework/Router/Router.php

@@ -21,7 +21,7 @@ class Router {
         ['mood', 'MoodController@handlePost', ['POST']],
         ['setup', 'AdminController@showSetup'],
         ['setup', 'AdminController@handleSetup', ['POST']],
-        ['tick/{y}/{m}/{d}/{h}/{i}/{s}', 'TickController'],
+        ['tick/{id}', 'TickController'],
         ['css/custom/{filename}.css', 'CssController@serveCustomCss'],
     ];
 

src/Framework/Util/Util.php

@@ -30,6 +30,8 @@ class Util {
 
     // For relative time display, compare the stored time to the current time
     // and display it as "X seconds/minutes/hours/days etc." ago
+    //
+    // TODO: Convert to either accepting a DateTime or use DateTime->fromFormat()
     public static function relative_time(string $tickTime): string {
         $datetime = new DateTime($tickTime);
         $now = new DateTime('now', $datetime->getTimezone());
@@ -52,15 +54,4 @@ class Util {
         }
         return $diff->s . ' second' . ($diff->s != 1 ? 's' : '') . ' ago';
     }
-
-    public static function tick_time_to_tick_path($tickTime){
-        [$date, $time] = explode(' ', $tickTime);
-        $dateParts = explode('-', $date);
-        $timeParts = explode(':', $time);
-
-        [$year, $month, $day] = $dateParts;
-        [$hour, $minute, $second] = $timeParts;
-
-        return "$year/$month/$day/$hour/$minute/$second";
-    }
 }

src/Model/TickModel/TickModel.php

@@ -1,105 +1,41 @@
 <?php
 class TickModel {
-    // Everything in this class just reads from and writes to the filesystem
-    // It doesn't maintain state, so everything's just a static function
-    public static function streamTicks(int $limit, int $offset = 0): Generator {
-        $tick_files = glob(TICKS_DIR . '/*/*/*.txt');
-        usort($tick_files, fn($a, $b) => strcmp($b, $a)); // sort filenames in reverse chronological order
+    public function stream(int $limit, int $offset = 0): Generator {
+        global $db;
 
-        $count = 0;
-        foreach ($tick_files as $file) {
-            // read all the ticks from the current file and reverse the order
-            // so the most recent ones are first
-            //
-            // each file is a single day, so we never hold more than
-            // one day's ticks in memory
-            $lines = array_reverse(
-                file($file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES)
-            );
+        $stmt = $db->prepare("SELECT id, timestamp, tick FROM tick ORDER BY timestamp DESC LIMIT ? OFFSET ?");
+        $stmt->execute([$limit, $offset]);
 
-            // split the path to the current file into the date components
-            $pathParts = explode('/', str_replace('\\', '/', $file));
-
-            // assign the different components to the appropriate part of the date
-            $year = $pathParts[count($pathParts) - 3];
-            $month = $pathParts[count($pathParts) - 2];
-            $day = pathinfo($pathParts[count($pathParts) - 1], PATHINFO_FILENAME);
-
-            foreach ($lines as $line) {
-                // just keep skipping ticks until we get to the starting point
-                if ($offset > 0) {
-                    $offset--;
-                    continue;
-                }
-
-                // Ticks are pipe-delimited: timestamp|text
-                // But just in case a tick contains a pipe, only split on the first one that occurs
-                list($time, $mood, $tick) = explode('|', $line, 3);
-
-                // Build the timestamp from the date and time
-                // Ticks are always stored in UTC
-                $timestampUTC = "$year-$month-$day $time";
-                yield [
-                    'timestamp' => $timestampUTC,
-                    'mood' => $mood,
-                    'tick' => $tick,
-                ];
-
-                if (++$count >= $limit) {
-                    return;
-                }
-            }
+        while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
+            yield [
+                'id' => $row['id'],
+                'timestamp' => $row['timestamp'],
+                'tick' => $row['tick'],
+            ];
         }
     }
 
-    public static function save(string $tick, string $mood=''): void {
-        // build the tick path and filename from the current time
-        $now = new DateTime('now', new DateTimeZone('UTC'));
+    public function insert(string $tick, ?DateTimeImmutable $datetime = null): void {
+        global $db;
+        $datetime ??= new DateTimeImmutable('now', new DateTimeZone('UTC'));
+        $timestamp = $datetime->format('Y-m-d H:i:s');
 
-        $year = $now->format('Y');
-        $month = $now->format('m');
-        $day = $now->format('d');
-        $time = $now->format('H:i:s');
-
-        // build the full path to the tick file
-        $dir = TICKS_DIR . "/$year/$month";
-        $filename = "$dir/$day.txt";
-
-        // create the directory if it doesn't exist
-        if (!is_dir($dir)) {
-            mkdir($dir, 0770, true);
-        }
-
-        // write the tick to the file (the file will be created if it doesn't exist)
-        $content = $time . '|' . $mood . '|' . $tick . "\n";
-        file_put_contents($filename, $content, FILE_APPEND);
+        $stmt = $db->prepare("INSERT INTO tick(timestamp, tick) values (?, ?)");
+        $stmt->execute([$timestamp, $tick]);
     }
 
-    public static function get(string $y, string $m, string $d, string $H, string $i, string $s): array{
-        $tickTime = new DateTime("$y-$m-$d $H:$i:$s");
-        $timestamp = "$H:$i:$s";
-        $file = TICKS_DIR . "/$y/$m/$d.txt";
+    public function get(int $id): array {
+        global $db;
 
-        if (!file_exists($file)) {
-            http_response_code(404);
-            echo "Tick not found: $file.";
-            exit;
-        }
+        $stmt = $db->prepare("SELECT timestamp, tick FROM tick WHERE id=?");
+        $stmt->execute([$id]);
+        $row = $stmt->fetch(PDO::FETCH_ASSOC);
 
-        $lines = file($file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
-        foreach ($lines as $line) {
-            if (str_starts_with($line, $timestamp)) {
-                echo $line;
-                exit;
-                list($time, $emoji, $tick) = explode('|', $line, 3);
-
-                return [
-                    'tickTime' => $tickTime,
-                    'emoji' => $emoji,
-                    'tick' => $tick,
-                    'config' => ConfigModel::load(),
-                ];
-            }
-        }
+        // TODO: Test for existence of row and handle absence.
+        return [
+            'tickTime' => $row['timestamp'],
+            'tick' => $row['tick'],
+            'config' => ConfigModel::load(),
+        ];
     }
 }

src/View/HomeView/HomeView.php

@@ -13,9 +13,6 @@ class HomeView {
                 ?>
                 <li class="tick" tabindex="0">
                     <time datetime="<?php echo $datetime->format('c') ?>"><?php echo Util::escape_html($relativeTime) ?></time>
-                    <?php if ($config->showTickMood): ?>
-                        <span><?php echo $tick['mood'] ?></span>
-                    <?php endif; ?>
                     <span class="tick-text"><?php echo Util::linkify(Util::escape_html($tick['tick'])) ?></span>
                 </li>
             <?php endforeach; ?>

templates/feed/atom.php

@@ -22,17 +22,8 @@ echo '<?xml version="1.0" encoding="utf-8"?>' . "\n";
         <name><?= Util::escape_xml($config->siteTitle) ?></name>
   </author>
 <?php foreach ($ticks as $tick):
-    // decompose the tick timestamp into the date/time parts
-    [$date, $time] = explode(' ', $tick['timestamp']);
-
-    $dateParts = explode('-', $date);
-    [$year, $month, $day] = $dateParts;
-
-    $timeParts = explode(':', $time);
-    [$hour, $minute, $second] = $timeParts;
-
     // build the tick entry components
-    $tickPath = "tick/$year/$month/$day/$hour/$minute/$second";
+    $tickPath = "tick/" . $tick['id'];
     $tickUrl = Util::escape_xml($siteUrl . $basePath . $tickPath);
     $tickTime = date(DATE_ATOM, strtotime($tick['timestamp']));
     $tickTitle = Util::escape_xml($tick['tick']);

templates/feed/rss.php

@@ -19,17 +19,9 @@ echo '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
     <language>en-us</language>
     <lastBuildDate><?php echo date(DATE_RSS); ?></lastBuildDate>
 <?php foreach ($ticks as $tick):
-    // decompose the tick timestamp into the date/time parts
-    [$date, $time] = explode(' ', $tick['timestamp']);
-
-    $dateParts = explode('-', $date);
-    [$year, $month, $day] = $dateParts;
-
-    $timeParts = explode(':', $time);
-    [$hour, $minute, $second] = $timeParts;
-
     // build the tick entry components
-    $tickPath = "tick/$year/$month/$day/$hour/$minute/$second";
+    //$tickPath = "tick/$year/$month/$day/$hour/$minute/$second";
+    $tickPath = "tick/" . $tick['id'];
     $tickUrl = Util::escape_xml($config->baseUrl . $config->basePath . $tickPath);
     $tickDate = date(DATE_RSS, strtotime($tick['timestamp']));
     $tickTitle = Util::escape_xml($tick['tick']);

templates/partials/tick.php

@@ -1,5 +1,4 @@
-<?php /** @var ConfigModel $config */ ?>
 <?php /** @var Date $tickTime */ ?>
 <?php /** @var string $tick */ ?>
-        <h1>Tick from <?= $tickTime->format('Y-m-d H:i:s'); ?></h1>
+        <h1>Tick from <?= $tickTime; ?></h1>
         <p><?= Util::linkify(Util::escape_html($tick)) ?></p>

tests/Framework/Util/UtilTest.php

@@ -1,31 +1,29 @@
 <?php
 use PHPUnit\Framework\TestCase;
+use PHPUnit\Framework\Attributes\DataProvider;
 
 final class UtilTest extends TestCase
 {
-    public function testCanDisplayRelativeTime(): void
-    {
+    // Define test date (strings) and expected outputs for
+    // testCanDisplayRelativeTime
+    public static function dateProvider(): array {
         $datetime = new DateTimeImmutable();
 
-        $oneMinuteAgo = $datetime->modify('-1 minute')->format('c');
-        $relativeTime = Util::relative_time($oneMinuteAgo);
-        $this->assertSame($relativeTime, "1 minute ago");
-
-        $twoHoursAgo = $datetime->modify('-2 hours')->format('c');
-        $relativeTime = Util::relative_time($twoHoursAgo);
-        $this->assertSame($relativeTime, "2 hours ago");
-
-        $threeDaysAgo = $datetime->modify('-3 days')->format('c');
-        $relativeTime = Util::relative_time($threeDaysAgo);
-        $this->assertSame($relativeTime, "3 days ago");
-
-        $fourMonthsAgo = $datetime->modify('-4 months')->format('c');
-        $relativeTime = Util::relative_time($fourMonthsAgo);
-        $this->assertSame($relativeTime, "4 months ago");
-
-        $fiveYearsAgo = $datetime->modify('-5 years')->format('c');
-        $relativeTime = Util::relative_time($fiveYearsAgo);
-        $this->assertSame($relativeTime, "5 years ago");
-
+        return [
+            '1 minute ago' => [$datetime->modify('-1 minute')->format('c'), '1 minute ago'],
+            '2 hours ago'  => [$datetime->modify('-2 hours')->format('c'), '2 hours ago'],
+            '3 days ago'   => [$datetime->modify('-3 days')->format('c'), '3 days ago'],
+            '4 months ago' => [$datetime->modify('-4 months')->format('c'), '4 months ago'],
+            '5 years ago'  => [$datetime->modify('-5 years')->format('c'), '5 years ago']
+        ];
     }
+
+    // Validate that the datetime strings provided by dateProvider
+    // yield the expected display strings
+    #[DataProvider('dateProvider')]
+    public function testCanDisplayRelativeTime(string $datetimeString, string $display): void {
+        $relativeTime = Util::relative_time($datetimeString);
+        $this->assertSame($relativeTime, $display);
+    }
+
 }