Rolling back a model

pip install pytest

"""Keeps a small, ordered history of releases so a rollback always goes to
the version that was actually active immediately before the current one --
not to the first version ever shipped.
"""
import time


class ReleaseHistory:
    def __init__(self):
        self.deployed_order = []   # every version, in the order it was deployed
        self.active_version = None
        self.log = []

    def deploy(self, version: str):
        self.deployed_order.append(version)
        previous = self.active_version
        self.active_version = version
        self.log.append({
            "time": round(time.time(), 3), "action": "deploy",
            "from": previous, "to": version, "reason": "manual release",
        })

    def rollback(self, reason: str):
        """Goes back to whichever version was active immediately before this one."""
        if len(self.deployed_order) < 2:
            raise RuntimeError("no previous version to roll back to")
        current = self.active_version
        previous = self.deployed_order[self.deployed_order.index(current) - 1]
        self.active_version = previous
        self.log.append({
            "time": round(time.time(), 3), "action": "rollback",
            "from": current, "to": previous, "reason": reason,
        })
        return previous


def monitor_and_maybe_rollback(history: ReleaseHistory, error_rate: float, threshold: float = 0.05):
    if error_rate > threshold:
        new_active = history.rollback(reason=f"error rate {error_rate:.3f} exceeded threshold {threshold}")
        print(f"AUTO-ROLLBACK: {history.log[-1]['from']} -> {new_active}  ({history.log[-1]['reason']})")
        return True
    print(f"healthy: error rate {error_rate:.3f} is within the {threshold} threshold, no action")
    return False

from release_history import *

history = ReleaseHistory()
history.deploy("v1")
history.deploy("v2")

# v2 turns out to have a real problem
monitor_and_maybe_rollback(history, error_rate=0.11, threshold=0.05)
print("active after monitoring:", history.active_version)

# later, v3 ships (a fixed version) and later ALSO needs a rollback --
# it must go back to v2, not all the way back to v1
history.deploy("v3")
monitor_and_maybe_rollback(history, error_rate=0.09, threshold=0.05)
print("active after second rollback:", history.active_version)

from release_history import *

history = ReleaseHistory()
history.deploy("v1")
history.deploy("v2")
history.rollback(reason="error rate 0.110 exceeded threshold 0.05")
history.deploy("v3")
history.rollback(reason="error rate 0.090 exceeded threshold 0.05")

for entry in history.log:
    print(entry)

import pytest
from release_history import ReleaseHistory, monitor_and_maybe_rollback


def test_rollback_goes_to_the_immediately_prior_version_not_the_first_one():
    history = ReleaseHistory()
    history.deploy("v1")
    history.deploy("v2")
    history.deploy("v3")
    history.rollback(reason="test")
    assert history.active_version == "v2"


def test_rollback_with_no_previous_version_raises_instead_of_silently_doing_nothing():
    history = ReleaseHistory()
    history.deploy("v1")
    with pytest.raises(RuntimeError):
        history.rollback(reason="test")


def test_a_bad_error_rate_triggers_an_automatic_rollback():
    history = ReleaseHistory()
    history.deploy("v1")
    history.deploy("v2")
    triggered = monitor_and_maybe_rollback(history, error_rate=0.20, threshold=0.05)
    assert triggered is True
    assert history.active_version == "v1"


def test_every_rollback_is_recorded_with_a_reason():
    history = ReleaseHistory()
    history.deploy("v1")
    history.deploy("v2")
    history.rollback(reason="manual: bad answers reported by support")
    last = history.log[-1]
    assert last["action"] == "rollback"
    assert "bad answers" in last["reason"]

pytest test_rollback.py -q