From d9113a61c2f1ee57139dfeb5ffa09c3fed60b26b Mon Sep 17 00:00:00 2001 From: zPlus Date: Wed, 27 Jul 2022 11:02:11 +0200 Subject: [PATCH] Initial commit. This is the first working version of CLIF. --- .gitignore | 2 + README | 149 +++++ emails.py | 238 ++++++++ requirements.txt | 8 + static/css/clif.css | 285 +++++++++ static/css/pygments_default.css | 80 +++ templates/about.html | 56 ++ templates/explore.html | 25 + templates/index.html | 19 + templates/layout.html | 21 + templates/mailing_list/emails.html | 18 + templates/mailing_list/emails_thread.html | 54 ++ templates/mailing_list/mailing_list.html | 11 + templates/repository/blob.html | 30 + templates/repository/log.html | 75 +++ templates/repository/readme.html | 36 ++ templates/repository/refs.html | 27 + templates/repository/repository.html | 30 + templates/repository/tree.html | 54 ++ web.py | 670 ++++++++++++++++++++++ web.service | 13 + 21 files changed, 1901 insertions(+) create mode 100644 .gitignore create mode 100644 README create mode 100755 emails.py create mode 100644 requirements.txt create mode 100644 static/css/clif.css create mode 100644 static/css/pygments_default.css create mode 100644 templates/about.html create mode 100644 templates/explore.html create mode 100644 templates/index.html create mode 100644 templates/layout.html create mode 100644 templates/mailing_list/emails.html create mode 100644 templates/mailing_list/emails_thread.html create mode 100644 templates/mailing_list/mailing_list.html create mode 100644 templates/repository/blob.html create mode 100644 templates/repository/log.html create mode 100644 templates/repository/readme.html create mode 100644 templates/repository/refs.html create mode 100644 templates/repository/repository.html create mode 100644 templates/repository/tree.html create mode 100644 web.py create mode 100644 web.service diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..82adb58 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +venv diff --git a/README b/README new file mode 100644 index 0000000..cb4bfb2 --- /dev/null +++ b/README @@ -0,0 +1,149 @@ +# Installation + +1. Install Gitolite + +Follow instructions at https://gitolite.com/gitolite/fool_proof_setup.html +When Gitolite is installed, clone the gitolite-admin repository and add this to +conf/gitolite.conf: + + repo CREATOR/..* + C = @all + RW+ = CREATOR + R = @all + +The rule above will allow any registered user (@all) to create a repository. CREATOR +is a gitolite keywords and it's replaced with the username who created the repo. +To create a new repository, just use "git clone git@domain:username/reponame". +Since the regexp CREATOR/..* will replace CREATOR with the user name, the logged +in user will be allowed to create new repositories *only* under their username. + +Adding new users is as simple as adding their key to gitolite-admin/keydir/.pub + +Gitolite does not do authentication, it only does authorization. The name of the +logged in user is provided as an environment variable. In order to allow anonymous +HTTP clones, ie. for allowing "git clone https://..." without any authentication, +the web app automatically sets a generic username value of "anonymous". We need +to let Gitolite know what the unauthenticated user is going to be called so that +it can check authorization. To do this, just add the following to ~/.gitolite.rc +in the section marked "rc variables used by various features". This is explained +at https://gitolite.com/gitolite/http.html#allowing-unauthenticated-access + + HTTP_ANON_USER => 'anonymous', + +Enable some non-core commands that are useful to us. This is done by editing ~/.gitolite.rc: + + 'ENABLE' => [ + ... existing commands + + # Allow to change HEAD reference (default branch) like this: + # ssh git@host symbolic-ref HEAD refs/heads/ + 'symbolic-ref', + ] + + +2. Emails + +Start by downloading the clif repository: + + git clone /home/git + +Change the settings inside the emails.py file. + +Add the following to /etc/postfix/main.cf. This will forward all emails to the +system user "git" + + luser_relay = git + local_recipient_maps = + +Then add the following to /home/git/.forward. ".forward" is a sendmail file, also +used by postfix, used for deciding how to deliver the message the the system user. +For our purposes, we instruct postfix to pipe all the emails for user "git" to our +script: + + |/home/git/clif/emails.py + +make sure the script is executable: + + chmod +x /home/git/clif/emails.py + +3. Web UI + +Start by downloading the clif repository: + + git clone /home/git + +Install the requirements: + + cd /home/git/clif + python3 -m venv venv + source venv/bin/activate + pip install -r requirements.txt + +Change the settings inside the web.py file. + +Install a SystemD service: + + cp web.service /etc/systemd/system/clif-web.service + systemctl daemon-reload + systemctl enable clif-web + systemctl start clif-web + +4. TLS certificate + +Now we create a new TLS certificate for supporting HTTPS connections: + + apt-get install certbot + certbot certonly --webroot -w /var/www/html -d domain.tld + +The cert is created in /etc/letsencrypt/live// + +Lighttpd requires the certificate and private key to be in a single file: + + cat privkey.pem cert.pem > privkey+cert.pem + +Configure lighttpd reverse proxy: + + vim /etc/lighttpd/lighttpd.conf + + server.modules += ( + "mod_fastcgi", + "mod_proxy", + ) + + $HTTP["scheme"] == "http" { + url.redirect = ("" => "https://${url.authority}${url.path}${qsa}") + url.redirect-code = 308 + } + + $SERVER["socket"] == ":443" { + ssl.engine = "enable" + ssl.pemfile = "/etc/letsencrypt/live//privkey+cert.pem" + ssl.ca-file = "/etc/letsencrypt/live//chain.pem" + + $HTTP["host"] == "" { + proxy.server = ( + "" => ( + ( "host" => "127.0.0.1", "port" => 5000 ) + ) + ) + # server.document-root = "/var/www/html" # Document Root + # server.errorlog = "/" + # accesslog.filename = "/" + } + } + +Let's Encrypt certificates expire every 90 days, so we need to setup a cron job +that will generate a new privkey+cert.pem file, and reload lighttpd too. + + vim /etc/cron.weekly/clif-letsencrypt + chmod +x /etc/cron.weekly/clif-letsencrypt + + certbot renew + cd /etc/letsencrypt/live/ + cat privkey.pem cert.pem > privkey+cert.pem + service lighttpd restart + +# Development + + gunicorn --reload --bind localhost:5000 web:application + diff --git a/emails.py b/emails.py new file mode 100755 index 0000000..01b2487 --- /dev/null +++ b/emails.py @@ -0,0 +1,238 @@ +#!/home/git/clif/venv/bin/python + +############################################################################### +# This script is called by Postfix every time it receives an email. +# This script will accept incoming emails and add them to the mailing lists repositories. +############################################################################### + +import datetime +import email +import email.policy +import hashlib +import json +import logging +import os +import pygit2 +import smtplib +import sys + + + + +############################################################################### +# SETTINGS +############################################################################### + +# Default options for the configuration file +configuration = { + 'enabled': True, + 'subscribers': [] +} + +# The "domain" part in address@domain that we're expecting to see. +# All emails addressed to another domain will be ignored. +SERVER_DOMAIN = 'domain.local' + +REPOSITORIES_PATH = '/home/git/repositories' + +# Level | Numeric value +# ---------|-------------- +# CRITICAL | 50 +# ERROR | 40 +# WARNING | 30 +# INFO | 20 +# DEBUG | 10 +# NOTSET | 0 +logging.basicConfig(filename='/home/git/clif/emails.log', + level=logging.NOTSET, + format='[%(asctime)s] %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S%z') + + + + +############################################################################### +# ACCEPT/VALIDATE INCOMING EMAIL +############################################################################### + +# Retrieve the email message from stdin (Postfix has piped this script) +message_raw = sys.stdin.read() +message = email.message_from_string(message_raw, policy=email.policy.default) + +email_id = message.get('message-id') +email_id_hash = hashlib.sha256(email_id.encode('utf-8')).hexdigest()[:8] # This will be used as thread ID +email_from = email.utils.parseaddr(message.get('from')) +email_to = email.utils.parseaddr(message.get('to')) +email_in_reply_to = message.get('in-reply-to') +email_subject = message.get('subject') +email_body = message.get_body(('plain',)).get_content() # Accept plaintext only! + +logging.info('Received email from {} to {} with subject {}'.format(email_from, email_to, email_subject)) + +if not email_id: + logging.info('Refuting email without a Message-ID: {}'.format(email_subject)) + exit() + +if not email_body: + logging.warning('Refuting email without plaintext body: {}'.format(email_subject)) + exit() + +if not email_to[1].endswith('@' + SERVER_DOMAIN): + logging.warning('Refuting email with bad recipient domain: {}'.format(email_to)) + exit() + +# Get the repository name. We use email addresses formatted as @SERVER_DOMAIN +repository_name = email_to[1].rsplit('@', 1)[0] +repository_path = os.path.join(REPOSITORIES_PATH, repository_name + '.mlist.git') + +if '..' in repository_name: + logging.warning('Refuting email because the repository name contains "..": {}'.format(repository_name)) + exit() + +if '/' not in repository_name: + logging.warning('Refuting email because the repository name does not contain a namespace: {}'.format(repository_name)) + exit() + + + + +############################################################################### +# ADD EMAIL TO USER REPOSITORY +############################################################################### + +if not os.path.isdir(repository_path): + logging.warning('Repository path does not exist: {}'.format(repository_path)) + exit() + +try: + repo = pygit2.Repository(repository_path) +except: + logging.warning('Not a valid repository: {}'.format(repository_path)) + exit() + +try: + head_tree = repo.revparse_single('HEAD').tree +except: + logging.warning('Could not find HEAD ref: {}'.format(repository_path)) + exit() + +try: + configuration = configuration | json.loads(head_tree['configuration'].data.decode('UTF-8')) +except: + logging.info('Could not load configuration file for repository {}. The default configuration will be used instead.'.format(repository_path)) + +if configuration['enabled'] == False: + logging.info('Ignoring incoming email for repository {} because emails are disabled.'.format(repository_path)) + exit() + +logging.debug('Accepting email from {} to {} with subject {}'.format(email_from, email_to, email_subject)) + +# At this point we need to add the incoming email to the repository. +# If the email is a reply (ie. it contains the In-Reply-To header, we retrieve the +# existing tree for the thread. Otherwise, we will create a new tree. + +thread_tree = None +thread_title = '{} {} {}'.format( + datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d %H:%M:%S'), + email_id_hash, + email_subject.replace('/', '-') +) + +if email_in_reply_to: + # The hash of the email that is being replied to + parent_message_hash = hashlib.sha256(email_in_reply_to.encode('utf-8')).hexdigest()[:8] + + # Find the thread (tree) containing the parent message + for obj in head_tree: + if obj.type_str == 'tree' and parent_message_hash + '.email' in obj: + thread_tree = obj + thread_title = obj.name + break + + # We only accept emails in reply to existing messages + if not thread_tree: + logging.debug('In-Reply-To ID not found in repository: {}'.format(email_in_reply_to)) + exit() + +# Add the new email BLOB to the git store +message_oid = repo.create_blob(message_raw) + +# Add the blob that we've just created to the thread tree +thread_tree_builder = repo.TreeBuilder(thread_tree) if thread_tree else repo.TreeBuilder() +thread_tree_builder.insert(email_id_hash + '.email', message_oid, pygit2.GIT_FILEMODE_BLOB) +thread_tree_oid = thread_tree_builder.write() + +# Add the thread tree to the HEAD tree +head_tree_builder = repo.TreeBuilder(head_tree) +head_tree_builder.insert(thread_title, thread_tree_oid, pygit2.GIT_FILEMODE_TREE) +head_tree_oid = head_tree_builder.write() + +repo.create_commit( + repo.head.name, # reference name + pygit2.Signature('CLIF', '-'), # author + pygit2.Signature('CLIF', '-'), # committer + 'New email.', # message + head_tree_oid, # tree of this commit + [ repo.head.target ] # parents commit +) + + + + +############################################################################### +# FORWARD EMAIL TO THREAD PARTICIPANTS AND TO LIST SUBSCRIBERS +############################################################################### + +# Remove duplicates, if any +participants = list(set(configuration['subscribers'])) + +# Find all the participants in the thread, ie. everyone that has sent an email +thread_tree = repo.get(thread_tree_oid) +for obj in thread_tree: + try: + obj_message = email.message_from_string(obj.data.decode('UTF-8'), policy=email.policy.default) + obj_email_from = email.utils.parseaddr(obj_message.get('from'))[1] + + if obj_email_from not in participants: + participants.append(obj_email_from) + except: + logging.warning('Could not parse file for searching participants: {}'.format(obj.name)) + +# Remove list address from participants in order to avoid forwarding to us +while email_to[1] in participants: + participants.remove(email_to[1]) + +# Modify some headers before forwarding. +# Note: we need to delete them first because the SMTP client will only accept one +# of them at most, but message[] is an append operator ("message" is an instance of +# email.message.EmailMessage()) +# https://docs.python.org/3/library/email.message.html#email.message.EmailMessage +# TODO Some ISPs add the client IP to the email headers. Should we remove *all* +# unnecessary headers instead? +for header in [ 'Sender', 'Reply-To', + 'List-Id', 'List-Subscribe', 'List-Unsubscribe', 'List-Post' ]: + del message[header] + +message['Sender'] = '{}@{}'.format(repository_name, SERVER_DOMAIN) +message['Reply-To'] = '{}@{}'.format(repository_name, SERVER_DOMAIN) +message['List-Id'] = '<{}@{}>'.format(repository_name, SERVER_DOMAIN) +# message['List-Subscribe'] = '<>' +# message['List-Unsubscribe'] = '<>' +message['List-Post'] = '<{}@{}>'.format(repository_name, SERVER_DOMAIN) + +# Forward email to participants +try: + smtp_client = smtplib.SMTP('localhost') + + # "The from_addr and to_addrs parameters are used to construct the message envelope + # used by the transport agents. sendmail does not modify the message headers in any way." + # - https://docs.python.org/3/library/smtplib.html#smtplib.SMTP.sendmail + smtp_client.sendmail( + '{}@{}'.format(repository_name, SERVER_DOMAIN), # Envelope From + participants, # Envelope To + str(message)) # Message + + logging.debug("Successfully sent emails.") +except Exception as e: + logging.error("Error sending emails.") + logging.error(str(e)) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..bbee4b3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +bottle +gunicorn +jinja2 +pygit2 +pygments +python-magic +pytz +timeago diff --git a/static/css/clif.css b/static/css/clif.css new file mode 100644 index 0000000..1713087 --- /dev/null +++ b/static/css/clif.css @@ -0,0 +1,285 @@ +* { + /* + margin: 0; + padding: 0; + */ + -moz-box-sizing: border-box; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +html, body { + font-family: Sans-Serif; +} + +blockquote { + background-color: #f8f8f8; + border-left: 5px solid #e9e9e9; + font-size: .85em; + margin: 1em 0; + padding: .5em 1em; +} + +blockquote cite { + color: #999; + display: block; + font-size: .8em; + margin-top: 1em; +} + +blockquote cite:before { + content: "\2014 \2009"; +} + +p { + margin: 0 0 10px 0; +} + +pre { + font-size: 1rem; + line-height: 1.5rem; +} + +.top_bar { + display: flex; + width: 100%; +} + + .top_bar > .path { + flex-basis: auto; + padding: 0 5rem 0 0; + } + + .top_bar > .context { + } + +.page_content { + margin: 2rem 0 5rem 0; +} + +.repository { + display: flex; + width: 100%; +} + + .repository > .readme { + flex: 70%; + font-family: monospace; + font-size: 1rem; + white-space: pre-wrap; + padding: 0 1rem 0 0; + } + + .repository > .overview { + flex: 30%; + padding: 0 0 0 1rem; + } + +/* The page menu at the top */ +ul.menu { + display: inline-block; + list-style: none; + margin: 0; + overflow: hidden; + padding: 0; + vertical-align: middle; +} + + ul.menu > a { + border-bottom: .2rem solid transparent; + color: black; + display: inline-block; + float: left; + margin: 0 1rem 0 0; + padding: .1rem 0; + text-decoration: none; + } + + ul.menu > a:hover { + border-bottom: .2rem solid #ccc; + } + + ul.menu > a.selected { + border-bottom: .2rem solid black; + } + +div.ref_title { + font-weight: bold; + margin-bottom: 1rem; +} + +div.ref_title:not(:first-child) { + margin-top: 4rem; +} + + div.ref_title ~ div.ref { + margin: 0 0 0 1rem; + } + +span.head_label { + background-color: #b9faca; + border-radius: .1rem; + border: 1px solid black; + font-size: .6rem; + margin: 0 0 0 1rem; + padding: .1rem; +} + +div.tree_list { +} + + div.tree_list > a { + color: black; + display: block; + margin: 0; + padding: .5rem; + text-decoration: none; + } + div.tree_list > a > pre { + margin: 0; + } + + div.tree_list > a:hover { + background-color: #e3ecfa; + } + +table.log { + border-spacing: 0; + width: 100%; +} + + table.log td { + padding: .2rem .5rem; + white-space: nowrap; + width: auto; + } + + table.log > thead { + font-weight: bold; + } + + table.log > thead td { + padding-bottom: 1rem; + } + + table.log > thead td.time { + text-align: right; + } + + table.log > tbody { + } + + table.log > tbody > tr:hover { + background-color: #e3ecfa; + } + + table.log > tbody td.short_id { + font-family: ui-monospace, monospace; + } + + table.log > tbody td.message { + width: 100%; + } + + table.log > tbody td.message details > .fulltext { + margin: 1rem 2rem; + white-space: pre-wrap; + } + + table.log > tbody td.time { + text-align: right; + } + +div.threads { + +} + + div.threads > div { + margin-bottom: 1rem; + } + + div.threads div.title { + font-weight: bold; + } + + div.threads div.subtitle { + color: #666; + font-size: 0.8rem; + padding-top: .5rem; + } + +.thread { +} + + .thread > .title { + font-size: 1.2rem; + font-weight: bold; + } + + .thread > .subtitle { + color: #666; + margin: 1rem 0 2rem 0; + } + + .thread > .content { + display: flex; + } + + .thread .messages { + flex: 70%; + padding: 1rem; + } + + .thread .message { + border-radius: .1rem; + margin-bottom: 1rem; + } + + .thread .message:not(:last-child) { + border-bottom: 1px solid #d0d0d0; + } + + .thread .message > .header { + font-size: .9rem; + padding: 1rem; + } + + .thread .message > .header > details > .headers { + margin-top: 1rem; + } + + .thread .message > .body { + padding: 1rem; + white-space: pre-wrap; + } + + .thread .info { + flex: 30%; + padding: 1rem; + } + +/* Alternate background color used when displaying table data */ +.striped > *:nth-child(even) { + background-color: #f8f8f8; +} + +/* Override some Pygments rules of the default style */ +.highlight { + background: none; +} + + .highlight .linenos { + border-right: 1px solid #aaa; + padding-right: .5rem; + } + + .highlight .linenos a, + .highlight .linenos a:hover, + .highlight .linenos a:visited { + color: gray; + text-decoration: none; + } + + .highlight .code { + padding-left: 1rem; + } diff --git a/static/css/pygments_default.css b/static/css/pygments_default.css new file mode 100644 index 0000000..23b7e0c --- /dev/null +++ b/static/css/pygments_default.css @@ -0,0 +1,80 @@ +/** + * CSS styles for highlighting text. + * This is the default color style used by Pygments. + * Generated using: + * pygmentize -S default -f html -a .highlight > pygments_default.css + */ + +td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } +span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } +td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } +span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } +.highlight .hll { background-color: #ffffcc } +.highlight { background: #f8f8f8; } +.highlight .c { color: #3D7B7B; font-style: italic } /* Comment */ +.highlight .err { border: 1px solid #FF0000 } /* Error */ +.highlight .k { color: #008000; font-weight: bold } /* Keyword */ +.highlight .o { color: #666666 } /* Operator */ +.highlight .ch { color: #3D7B7B; font-style: italic } /* Comment.Hashbang */ +.highlight .cm { color: #3D7B7B; font-style: italic } /* Comment.Multiline */ +.highlight .cp { color: #9C6500 } /* Comment.Preproc */ +.highlight .cpf { color: #3D7B7B; font-style: italic } /* Comment.PreprocFile */ +.highlight .c1 { color: #3D7B7B; font-style: italic } /* Comment.Single */ +.highlight .cs { color: #3D7B7B; font-style: italic } /* Comment.Special */ +.highlight .gd { color: #A00000 } /* Generic.Deleted */ +.highlight .ge { font-style: italic } /* Generic.Emph */ +.highlight .gr { color: #E40000 } /* Generic.Error */ +.highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ +.highlight .gi { color: #008400 } /* Generic.Inserted */ +.highlight .go { color: #717171 } /* Generic.Output */ +.highlight .gp { color: #000080; font-weight: bold } /* Generic.Prompt */ +.highlight .gs { font-weight: bold } /* Generic.Strong */ +.highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ +.highlight .gt { color: #0044DD } /* Generic.Traceback */ +.highlight .kc { color: #008000; font-weight: bold } /* Keyword.Constant */ +.highlight .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */ +.highlight .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */ +.highlight .kp { color: #008000 } /* Keyword.Pseudo */ +.highlight .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */ +.highlight .kt { color: #B00040 } /* Keyword.Type */ +.highlight .m { color: #666666 } /* Literal.Number */ +.highlight .s { color: #BA2121 } /* Literal.String */ +.highlight .na { color: #687822 } /* Name.Attribute */ +.highlight .nb { color: #008000 } /* Name.Builtin */ +.highlight .nc { color: #0000FF; font-weight: bold } /* Name.Class */ +.highlight .no { color: #880000 } /* Name.Constant */ +.highlight .nd { color: #AA22FF } /* Name.Decorator */ +.highlight .ni { color: #717171; font-weight: bold } /* Name.Entity */ +.highlight .ne { color: #CB3F38; font-weight: bold } /* Name.Exception */ +.highlight .nf { color: #0000FF } /* Name.Function */ +.highlight .nl { color: #767600 } /* Name.Label */ +.highlight .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */ +.highlight .nt { color: #008000; font-weight: bold } /* Name.Tag */ +.highlight .nv { color: #19177C } /* Name.Variable */ +.highlight .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */ +.highlight .w { color: #bbbbbb } /* Text.Whitespace */ +.highlight .mb { color: #666666 } /* Literal.Number.Bin */ +.highlight .mf { color: #666666 } /* Literal.Number.Float */ +.highlight .mh { color: #666666 } /* Literal.Number.Hex */ +.highlight .mi { color: #666666 } /* Literal.Number.Integer */ +.highlight .mo { color: #666666 } /* Literal.Number.Oct */ +.highlight .sa { color: #BA2121 } /* Literal.String.Affix */ +.highlight .sb { color: #BA2121 } /* Literal.String.Backtick */ +.highlight .sc { color: #BA2121 } /* Literal.String.Char */ +.highlight .dl { color: #BA2121 } /* Literal.String.Delimiter */ +.highlight .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */ +.highlight .s2 { color: #BA2121 } /* Literal.String.Double */ +.highlight .se { color: #AA5D1F; font-weight: bold } /* Literal.String.Escape */ +.highlight .sh { color: #BA2121 } /* Literal.String.Heredoc */ +.highlight .si { color: #A45A77; font-weight: bold } /* Literal.String.Interpol */ +.highlight .sx { color: #008000 } /* Literal.String.Other */ +.highlight .sr { color: #A45A77 } /* Literal.String.Regex */ +.highlight .s1 { color: #BA2121 } /* Literal.String.Single */ +.highlight .ss { color: #19177C } /* Literal.String.Symbol */ +.highlight .bp { color: #008000 } /* Name.Builtin.Pseudo */ +.highlight .fm { color: #0000FF } /* Name.Function.Magic */ +.highlight .vc { color: #19177C } /* Name.Variable.Class */ +.highlight .vg { color: #19177C } /* Name.Variable.Global */ +.highlight .vi { color: #19177C } /* Name.Variable.Instance */ +.highlight .vm { color: #19177C } /* Name.Variable.Magic */ +.highlight .il { color: #666666 } /* Literal.Number.Integer.Long */ diff --git a/templates/about.html b/templates/about.html new file mode 100644 index 0000000..6274389 --- /dev/null +++ b/templates/about.html @@ -0,0 +1,56 @@ +{% extends "index.html" %} + +{% block menu_about_class %}selected{% endblock %} + +{% block content %} + +

+ Welcome! You have reached a public instance of the CLI Forge. +

+ +

+ This is a place for collaborative software development. It offers hosting for + Git repositories and mailing lists, and it aims at being entirely usable from + your CLI by leveraging existing tools and an email-driven workflow. + Read on for a quick introduction to using this instace. +

+ +
+ +

+ New users +
     + You only need an account if you wish to host your repositories on this instance. + Collaboration is done on mailing lists and you don't need an account for that. + When new users join the instance, they are automatically assigned the namespace + /<username> under which they can add new repositories.
+ Since this instance is still under testing it is not currently open for public + registrations, but you can still get an account by asking in #peers at irc.libera.chat. +

+ +

+ Adding repositories +
     + If you have an account, you can use CLIF as a remote for sharing your repositories. + Simply running git clone git@{{ domain }}:<namespace>/<repository> + will create a new empty repository that you can add to your list of remotes. +

+ +

+ Mailing lists +
     + Mailing lists are where collaboration happens, and they are stored in repositories + too. All you have to do in order to create a new mailing list is to create + a new repository with the suffix .mlist, for example + git clone git@{{ domain }}:alice/project.mlist. CLIF then will + begin accepting emails for alice/project@{{ domain }} and store + them inside the alice/project.mlist repository. +
+ New threads are created simply by sending a new email to the list address. + An account is not required. + It is also possible to join an existing thread by sending an email containing + the header In-Reply-To: <Message-ID>, where Message-ID + is the ID value of any previous message. +

+ +{% endblock %} diff --git a/templates/explore.html b/templates/explore.html new file mode 100644 index 0000000..751dd60 --- /dev/null +++ b/templates/explore.html @@ -0,0 +1,25 @@ +{% extends "index.html" %} + +{% block menu_explore_class %}selected{% endblock %} + +{% block content %} + +
+ On this instance: +
+ +
+ + {% for repository in repositories %} +
+ {% if repository.endswith('.mlist.git') %} + L   + {{ repository[:-4] }} + {% else %} + R   + {{ repository }} + {% endif %} +
+ {% endfor %} + +{% endblock %} diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..3654fed --- /dev/null +++ b/templates/index.html @@ -0,0 +1,19 @@ +{% extends "layout.html" %} + +{% block path %} +CLIF +{% endblock %} + +{% block context %} + +{% endblock %} diff --git a/templates/layout.html b/templates/layout.html new file mode 100644 index 0000000..93aaf83 --- /dev/null +++ b/templates/layout.html @@ -0,0 +1,21 @@ + + + + + + + + + + {{ title if title }} + + + +
+
{% block path %}{% endblock %}
+
{% block context %}{% endblock %}
+
+ +
{% block content %}{% endblock %}
+ + diff --git a/templates/mailing_list/emails.html b/templates/mailing_list/emails.html new file mode 100644 index 0000000..0b55ebf --- /dev/null +++ b/templates/mailing_list/emails.html @@ -0,0 +1,18 @@ +{% extends "mailing_list/mailing_list.html" %} + +{% block content %} + +
+ {% for thread in threads %} +
+
+ {{ thread.title }} +
+
+ Created {{ thread.datetime|ago }} +
+
+ {% endfor %} +
+ +{% endblock %} diff --git a/templates/mailing_list/emails_thread.html b/templates/mailing_list/emails_thread.html new file mode 100644 index 0000000..4af90c8 --- /dev/null +++ b/templates/mailing_list/emails_thread.html @@ -0,0 +1,54 @@ +{% extends "mailing_list/mailing_list.html" %} + +{% block content %} +
+ +
+ {{ thread.title }} +
+
+ #{{ thread.id }} - Created {{ thread.datetime|ago }} +
+ +
+ +
+ {% for email in emails %} +
+
+
+ + {{ email.from[0] }} <{{ email.from[1] }}> + {{ email.received_at|ago }} + +
+
+ Message-ID: {{ email.id }} +
+ {% if email.in_reply_to %} +
+ In-Reply-To: {{ email.in_reply_to }} +
+ {% endif %} +
Subject: {{ email.subject }}
+
+
+
+
{{ email.body }}
+
+ {% endfor %} +
+ +
+
+ {{ participants|length }} participants + {% for address in participants %} + {{ address[0] }} <{{ address[1] }}> + {% endfor %} +
+
+ +
+
+ +{% endblock %} diff --git a/templates/mailing_list/mailing_list.html b/templates/mailing_list/mailing_list.html new file mode 100644 index 0000000..d2611f4 --- /dev/null +++ b/templates/mailing_list/mailing_list.html @@ -0,0 +1,11 @@ +{% extends "layout.html" %} + +{% block path %} + home / + {{ list_address }} +{% endblock %} + +{% block context %} +
    +
+{% endblock %} diff --git a/templates/repository/blob.html b/templates/repository/blob.html new file mode 100644 index 0000000..9bf4cc7 --- /dev/null +++ b/templates/repository/blob.html @@ -0,0 +1,30 @@ +{% extends "repository/repository.html" %} + +{% block menu_tree_class %}selected{% endblock %} + +{% block content %} + +
+ ID: {{ blob.id }} +
+ {% if not blob.is_binary %} + {# 10 is the ASCII integer value of \n + Use 10 instead of \n because blob.data is a byte array, not a string. + #} + {{ blob.data.count(10) }} lines + {% else %} + binary + {% endif %} โ€” + {{ blob.size|human_size(B=True) }} โ€” + View raw +
+ +

+ + {% if blob.is_binary %} + Cannot display binary object. + {% else %} + {{ blob_formatted|safe }} + {% endif %} + +{% endblock %} diff --git a/templates/repository/log.html b/templates/repository/log.html new file mode 100644 index 0000000..83a0ec6 --- /dev/null +++ b/templates/repository/log.html @@ -0,0 +1,75 @@ +{% extends "repository/repository.html" %} + +{% block menu_log_class %}selected{% endblock %} + +{% block content %} + +
+
+ + +
+
+ +
+ + + + + + + + + + + + + {% for commit in commits %} + + + + + + + {% endfor %} + +
AuthorIDMessageCommit time
+ {% if commit.author.name|length > 0 %} + {{ commit.author.name }} + {% else %} + [anonymous] + {% endif %} + + {{ commit.short_id }} + + {% if commit.message|length <= 100 %} + {{ commit.message }} + {% else %} +
+ {{ commit.message[:100] }} ... +
{{ commit.message }}
+
+ {% endif %} + 		+ {{ commit_time(commit.commit_time, commit.commit_time_offset)|ago }} +
+ +
+ +{% endblock %} diff --git a/templates/repository/readme.html b/templates/repository/readme.html new file mode 100644 index 0000000..30a7fdd --- /dev/null +++ b/templates/repository/readme.html @@ -0,0 +1,36 @@ +{% extends "repository/repository.html" %} + +{% block menu_readme_class %}selected{% endblock %} + +{% block content %} + +
+ + {% if readme %} +
{{ readme }}
+ {% else %} +
This repository does not have a README.
+ {% endif %} + +
+ Anon. clone
+ https://{{ instance_domain }}/{{ repository[0] }}/{{ repository[1] }}.git + +

+ + SSH
+ git@{{ instance_domain }}:{{ repository[0] }}/{{ repository[1] }} + + {% if head_ref %} +

+ HEAD: {{ head_ref }} + {% endif %} + +

+ + Size: {{ repository_size }} +
+ +
+ +{% endblock %} diff --git a/templates/repository/refs.html b/templates/repository/refs.html new file mode 100644 index 0000000..0344ec6 --- /dev/null +++ b/templates/repository/refs.html @@ -0,0 +1,27 @@ +{% extends "repository/repository.html" %} + +{% block menu_refs_class %}selected{% endblock %} + +{% block content %} + +
Heads
+ + {% for branch in heads %} +
+ {{ branch[11:] }} + + {% if branch == head %} + HEAD + {% endif %} +
+ {% endfor %} + +
Tags
+ + {% for tag in tags %} +
+ {{ tag[10:] }} +
+ {% endfor %} + +{% endblock %} diff --git a/templates/repository/repository.html b/templates/repository/repository.html new file mode 100644 index 0000000..79abba3 --- /dev/null +++ b/templates/repository/repository.html @@ -0,0 +1,30 @@ +{% extends "layout.html" %} + +{% block path %} + home ยป + {{ repository }} +{% endblock %} + +{% block context %} + +{% endblock %} diff --git a/templates/repository/tree.html b/templates/repository/tree.html new file mode 100644 index 0000000..7679756 --- /dev/null +++ b/templates/repository/tree.html @@ -0,0 +1,54 @@ +{% extends "repository/repository.html" %} + +{% block menu_tree_class %}selected{% endblock %} + +{% block content %} + +
+
+ + +
+
+ +
+ {# Display folders (trees) first #} + + {% if tree_path %} + + 
                       ๐Ÿ • ..
+ + {% endif %} + + {# Display all the trees first #} + {% for obj in tree if obj.type_str == "tree": %} + + 
{{ obj.short_id }} {{ obj.filemode|filemode }}    ๐Ÿ— {{ obj.name }}
+ + {% endfor %} + + {# Display all other files #} + {% for obj in tree if obj.type_str != "tree": %} + + 
{{ obj.short_id }} {{ obj.filemode|filemode }} {{ obj.size|human_size }} {{ obj.name }}
+ + {% endfor %} +
+ +{% endblock %} diff --git a/web.py b/web.py new file mode 100644 index 0000000..38edbcf --- /dev/null +++ b/web.py @@ -0,0 +1,670 @@ +import bottle +from bottle import jinja2_template as template, request, response + +import datetime +import email +import email.policy +import functools +import glob +import hashlib +import magic +import os +import pathlib +import pygit2 +import pytz +import re +import stat +import subprocess +import sys +import timeago + +from pygments import highlight +from pygments.lexers import guess_lexer, guess_lexer_for_filename +from pygments.formatters import HtmlFormatter + + + + +############################################################################### +# SETTINGS +############################################################################### + +# The root folder where Gitolite stores the repositories. This is used to find the +# actual repositories. +GITOLITE_REPOSITORIES_ROOT = '/home/git/repositories' + +# These are only used when anonymous cloning over HTTPS +GITOLITE_SHELL = '/home/git/bin/gitolite-shell' +GITOLITE_HTTP_HOME = '/home/git' + +# The domain of this instance. This is only really used when displaying list addresses, +# or when the domain needs to be displayed on some pages. +INSTANCE_DOMAIN = 'domain.local' + + + + +############################################################################### +# UTILITY FUNCTIONS +# This is code that is reused several times within the Bottle controllers +# below, so it's been grouped into functions. +############################################################################### + +def list_repositories(): + repositories = [] + + # When topdown is True, the caller can modify the dirnames list in-place and + # walk() will only recurse into the subdirectories whose names remain in dirnames; + # this can be used to prune the search. + # https://docs.python.org/3.12/library/os.html#os.walk + for path, dirs, files in os.walk(GITOLITE_REPOSITORIES_ROOT, topdown=True): + # Remove all files, we only want to recurse into directories + files.clear() + + # This path is a git repo. Remove all sub-dirs because we don't need to + # recurse any further + if path.endswith('.git'): + dirs.clear() + + repository = os.path.relpath(path, GITOLITE_REPOSITORIES_ROOT) + + # DO NOT LIST gitolite-admin repository! + # This is the administration repository of this instance! + if repository.lower() == 'gitolite-admin.git': + continue + + repositories.append(repository) + + repositories.sort() + return repositories + + + + +############################################################################### +# WEB APP +# Here below are all the Bottle routes and controllers. +############################################################################### + +if not os.path.isdir(GITOLITE_REPOSITORIES_ROOT): + print('Invalid repositories path: {}'.format(GITOLITE_REPOSITORIES_ROOT)) + sys.exit() + +# This only exists for exporting the bottle app object for a WSGI server such as Gunicorn +application = bottle.app() + +# Directories to search for HTML templates +bottle.TEMPLATE_PATH = [ './templates' ] + +def human_size(bytes, B=False): + """ + Convert a file size in bytes to a human friendly form. + This is only used in templates when showing file sizes. + """ + + for unit in [ 'B' if B else '', 'K', 'M', 'G', 'T', 'P' ]: + if bytes < 1024: break + bytes = bytes / 1024 + + return '{}{}'.format(round(bytes), unit).rjust(5) + +def humanct(commit_time, commit_time_offset = 0): + """ + The following will add custom functions to the jinja2 template engine. + These will be available to use within templates. + """ + + delta = datetime.timedelta(minutes=commit_time_offset) + tz = datetime.timezone(delta) + + dt = datetime.datetime.fromtimestamp(commit_time, tz) + + return dt.astimezone(pytz.utc).strftime('%Y-%m-%d %H:%M:%S') + +template = functools.partial(template, template_settings = { + 'filters': { + 'ago': timeago.format, + 'datetime': lambda date: dateutil.parser.parse(date).strftime('%b %-d, %Y - %H:%M%z%Z'), + # Convert a file's mode to a string of the form '-rwxrwxrwx' + 'filemode': stat.filemode, + # Human-friendly file size: + 'human_size': human_size, + }, + 'globals': { + 'commit_time': humanct, + 'instance_domain': INSTANCE_DOMAIN, + 'now': lambda: datetime.datetime.now(datetime.timezone.utc), + 'request': request, + 'url': application.get_url, + }, + 'autoescape': True +}) + +@bottle.error(404) +def error404(error): + """ + Custom 404 page. + + :param error: bottle.HTTPError given by Bottle when calling abort(404). + """ + + return '[404] {}'.format(error.body) + +@bottle.get('/static/', name='static') +def static(filename): + """ + Path for serving static files. + """ + + return bottle.static_file(filename, root='./static/') + +@bottle.get('/', name='about') +def about(): + """ + The home page displayed at https://domain/ + """ + + return template('about.html', domain=INSTANCE_DOMAIN) + +@bottle.get('/explore', name='explore') +def explore(): + """ + The home page displayed at https://domain/ + """ + + repositories = list_repositories() + + return template('explore.html', repositories=repositories) + +@bottle.get('/.git', name='readme') +def readme(repository): + """ + Show README of the repository. + + :param repository: Match repository name ending with ".git" + """ + + repository += '.git' + path = os.path.join(GITOLITE_REPOSITORIES_ROOT, repository) + + if not os.path.isdir(path): + bottle.abort(404, 'No repository at this path.') + + repo = pygit2.Repository(path) + local_branches = list(repo.branches.local) + + HEAD = None + ref_name = None + + try: + HEAD = repo.head.name + ref_name = HEAD + except: + for name_candidate in [ 'master', 'main', 'trunk', 'development', 'dev' ]: + if name_candidate in local_branches: + ref_name = name_candidate + break + + readme = '' + + if ref_name: + tree = repo.revparse_single(ref_name).tree + + for e in tree: + if e.name.lower() not in [ 'readme', 'readme.md', 'readme.rst' ]: + continue + + if e.is_binary: + continue + + # Read the README content, cut at 1MB + readme = tree[e.name].data[:1048576].decode('UTF-8') + break + + repo_size = sum(f.stat().st_size for f in pathlib.Path(path).glob("**/*")) + + return template('repository/readme.html', + readme=readme, + repository=repository, + repository_size=human_size(repo_size), + head_ref=HEAD) + +@bottle.get('/.git/refs', name='refs') +def refs(repository): + """ + List repository refs + """ + + repository += '.git' + path = os.path.join(GITOLITE_REPOSITORIES_ROOT, repository) + + if not os.path.isdir(path): + bottle.abort(404, 'No repository at this path.') + + repo = pygit2.Repository(path) + + if repo.is_empty: + return template('repository/refs.html', + repository=repository) + + HEAD = None + heads = [] + tags = [] + + for ref in repo.references: + if ref.startswith('refs/heads/'): + heads.append(ref) + if ref.startswith('refs/tags/'): + tags.append(ref) + + heads.sort() + tags.sort() + + try: + HEAD = repo.head.name + except: + pass + + return template('repository/refs.html', + repository=repository, + heads=heads, tags=tags, head=HEAD) + +@bottle.get('/.git/tree/', name='tree') +@bottle.get('/.git/tree//', name='tree_path') +def tree(repository, revision, tree_path=None): + """ + Show commit tree. + """ + + repository += '.git' + repository_path = os.path.join(GITOLITE_REPOSITORIES_ROOT, repository) + + if not os.path.isdir(repository_path): + bottle.abort(404, 'No repository at this path.') + + repo = pygit2.Repository(repository_path) + + if repo.is_empty: + return template('repository/tree.html', + repository=repository, revision=revision) + + git_object = None + + try: + git_object = repo.revparse_single(revision) + except: + pass + + if not git_object: + return template('repository/tree.html', + repository=repository, revision=revision) + + # List all the references. + # This is used for allowing the user to switch revision with a selector. + HEAD = None + heads = [] + tags = [] + for ref in repo.references: + if ref.startswith('refs/heads/'): heads.append(ref) + if ref.startswith('refs/tags/'): tags.append(ref) + heads.sort() + tags.sort() + + try: + HEAD = repo.head.name + except: + pass + + if git_object.type == pygit2.GIT_OBJ_TAG: + git_object = git_object.peel(None) + + if git_object.type == pygit2.GIT_OBJ_COMMIT: + git_object = git_object.tree + + if git_object.type == pygit2.GIT_OBJ_TREE and tree_path: + git_object = git_object[tree_path] + + if git_object.type == pygit2.GIT_OBJ_TREE: + return template( + 'repository/tree.html', + heads=heads, head_ref=HEAD, tags=tags, + tree=git_object, + tree_path=tree_path, + repository=repository, revision=revision) + + if git_object.type == pygit2.GIT_OBJ_BLOB: + + # Highlight blob text + if git_object.is_binary: + blob_formatted = '' + else: + blob_data = git_object.data.decode('UTF-8') + + # Guess Pygments lexer by filename, or by content if we can't find one + try: + pygments_lexer = guess_lexer_for_filename(git_object.name, blob_data) + except: + pygments_lexer = guess_lexer(blob_data) + + pygments_formatter = HtmlFormatter(nobackground=True, linenos=True, anchorlinenos=True, + lineanchors='line') + + blob_formatted = highlight(blob_data, pygments_lexer, pygments_formatter) + + return template( + 'repository/blob.html', + heads=heads, tags=tags, + blob=git_object, + blob_formatted=blob_formatted, + repository=repository, revision=revision, + tree_path=tree_path) + + bottle.abort(404) + +@bottle.post('/.git/tree', name='tree_change') +def tree_change(repository): + """ + Switch revision in tree page. + This route is used by the
in the "tree page when changing the revision + to be displayed. + """ + + revision = request.forms.get('revision') + + bottle.redirect(application.get_url('tree', + repository=repository, + revision=revision)) + +@bottle.get('/.git/log/', name='log') +def log(repository, revision): + """ + Show commit log. + """ + + repository += '.git' + repository_path = os.path.join(GITOLITE_REPOSITORIES_ROOT, repository) + + if not os.path.isdir(repository_path): + bottle.abort(404, 'No repository at this path.') + + repo = pygit2.Repository(repository_path) + + if repo.is_empty: + return template('repository/log.html', + repository=repository, revision=revision) + + git_object = None + + try: + git_object = repo.revparse_single(revision) + except: + pass + + if not git_object: + return template('repository/log.html', + repository=repository, revision=revision) + + # List all the references. + # This is used for allowing the user to switch revision with a selector. + HEAD = None + heads = [] + tags = [] + for ref in repo.references: + if ref.startswith('refs/heads/'): heads.append(ref) + if ref.startswith('refs/tags/'): tags.append(ref) + heads.sort() + tags.sort() + + try: + HEAD = repo.head.name + except: + pass + + if git_object.type in [ pygit2.GIT_OBJ_TREE, pygit2.GIT_OBJ_BLOB ]: + return 'Not a valid ref' + + if git_object.type == pygit2.GIT_OBJ_TAG: + git_object = git_object.peel(None) + + # At this point git_object should be a valid pygit2.GIT_OBJ_COMMIT + + # Read 50 commits + commits = [] + for commit in repo.walk(git_object.id): + commits.append(commit) + if len(commits) >= 50: + break + + return template( + 'repository/log.html', + heads=heads, head_ref=HEAD, tags=tags, + commits=commits, + repository=repository, revision=revision) + +@bottle.get('/.git/raw//', name='raw') +def raw(repository, revision, tree_path): + """ + Return a raw blow object. + """ + + repository += '.git' + repository_path = os.path.join(GITOLITE_REPOSITORIES_ROOT, repository) + + if not os.path.isdir(repository_path): + bottle.abort(404, 'No repository at this path.') + + repo = pygit2.Repository(repository_path) + + if repo.is_empty: + return "" + + git_tree = None + + try: + git_object = repo.revparse_single(revision) + except: + pass + + if not git_object or git_object.type != pygit2.GIT_OBJ_COMMIT: + bottle.abort(404, 'Not a valid revision.') + + blob = None + + try: + blob = git_object.tree[tree_path] + except: + bottle.abort(404, 'Object does not exist.') + + if blob.type != pygit2.GIT_OBJ_BLOB: + bottle.abort(404, 'Object is not a blob.') + + mime = magic.from_buffer(blob.data[:1048576], mime=True) + response.content_type = mime + return blob.data + +@bottle.get('/.git/info/refs') +@bottle.post('/.git/git-upload-pack') +def git_smart_http(repository): + """ + This controller proxies Git Smart HTTP requests to gitolite-shell for allowing + anonymous clones over HTTP. Looks like anonymous clones are not possible via SSH, + hence why we have this feature. + Note that this controller only matches "git-upload-pack" (used for fetching) + but does not match "git-receive-pack" (used for pushing). Pushing should only + happen via SSH. + + Note: If CLIF is running behind a web server such as httpd or lighttpd, the + same behavior of this controller can be achieved much more simply by configuring + the server with CGI and an alias that redirects the URLs above to the gitolite-shell + script. However, this controller exists so that anonymous HTTP clones can work + "out of the box" without any manual configuration of the server. + + Documentation useful for understanding how this works: + https://git-scm.com/docs/http-protocol + https://bottlepy.org/docs/dev/async.html + https://gitolite.com/gitolite/http.html#allowing-unauthenticated-access + """ + + # Environment variables for the Gitolite shell + # TODO Gitolite gives a warning: "WARNING: Use of uninitialized value in concatenation (.) or string at /home/git/bin/gitolite-shell line 239" + # Looks like some non-critical env vars are missing here: REMOTE_PORT SERVER_ADDR SERVER_PORT + gitenv = { + **os.environ, + + # https://git-scm.com/docs/git-http-backend#_environment + 'PATH_INFO': request.path, + 'REMOTE_USER': 'anonymous', # This user must be set in ~/.gitolite.rc like this: + # HTTP_ANON_USER => 'anonymous', + 'REMOTE_ADDR': request.remote_addr, + 'CONTENT_TYPE': request.content_type, + 'QUERY_STRING': request.query_string, + 'REQUEST_METHOD': request.method, + 'GIT_PROJECT_ROOT': GITOLITE_REPOSITORIES_ROOT, + 'GIT_HTTP_EXPORT_ALL': 'true', + + # Additional variables required by Gitolite + 'REQUEST_URI': request.fullpath, + 'GITOLITE_HTTP_HOME': GITOLITE_HTTP_HOME, + 'HOME': GITOLITE_HTTP_HOME, + } + + # Start a Gitolite shell. + # Do not replace .Popen() with .run() because it waits for child process to finish before returning. + proc = subprocess.Popen( + [ GITOLITE_SHELL ], + env = gitenv, + stdin = subprocess.PIPE, + stdout = subprocess.PIPE) + # stderr = ) + + # Write the whole request body to Gitolite stdin. + # Don't forget to close the pipe or it will hang! + proc.stdin.write(request.body.read()) + proc.stdin.close() + + # Now we process the Gitolite response and return it to the client. + + # First we need to scan all the HTTP headers in the response so that we can + # add them to the bottle response... + for line in proc.stdout: + line = line.decode('UTF-8').strip() + + # Empty line means no more headers + if line == '': + break + + header = line.split(':', 1) + response.set_header(header[0].strip(), header[1].strip()) + + # ...then we can return the rest of the Gitolite response to the client as we read it + for line in proc.stdout: + yield line + +@bottle.get('/.mlist', name='threads') +def threads(repository): + """ + List email threads. + + :param repository: Match repository name NOT ending with ".git" + """ + + path = os.path.join(GITOLITE_REPOSITORIES_ROOT, repository + '.mlist.git') + list_address = '{}@{}'.format(repository, INSTANCE_DOMAIN) + + if not os.path.isdir(path): + bottle.abort(404, 'No repository at this path.') + + repo = pygit2.Repository(path) + tree = repo.revparse_single('HEAD').tree + + threads_list = [] + + for obj in tree: + if obj.type != pygit2.GIT_OBJ_TREE: + continue + + threads_list.append(obj.name) + + threads_list.sort(reverse=True) + + for i in range(0, len(threads_list)): + thread_date, thread_time, thread_id, thread_title = threads_list[i].split(' ', 3) + + threads_list[i] = { + 'datetime': thread_date + ' ' + thread_time, + 'id': thread_id, + 'title': thread_title + } + + return template('mailing_list/emails.html', threads=threads_list, + list_address=list_address, + repository=repository) + +@bottle.get('/.mlist/', name='thread') +def thread(repository, thread_id): + """ + Show a single email thread. + """ + + path = os.path.join(GITOLITE_REPOSITORIES_ROOT, repository + '.mlist.git') + list_address = '{}@{}'.format(repository, INSTANCE_DOMAIN) + + if not os.path.isdir(path): + bottle.abort(404, 'No repository at this path.') + + repo = pygit2.Repository(path) + head_tree = repo.revparse_single('HEAD').tree + thread_tree = None + + for obj in head_tree: + if obj.type != pygit2.GIT_OBJ_TREE: + continue + + if thread_id in obj.name: + thread_tree = obj + break + + if not thread_tree: + bottle.abort(404, 'Not a valid thread') + + thread_date, thread_time, thread_id, thread_title = thread_tree.name.split(' ', 3) + thread_data = { + 'datetime': thread_date + ' ' + thread_time, + 'id': thread_id, + 'title': thread_title + } + + # Read all the emails in this thread and collect some statistics on the way (for + # displaying purposes only) + emails = [] + participants = [] + + for obj in thread_tree: + if obj.type != pygit2.GIT_OBJ_BLOB \ + or not obj.name.endswith('.email'): + continue + + message = email.message_from_string(obj.data.decode('UTF-8'), policy=email.policy.default) + + email_data = { + 'id': message.get('message-id'), + 'id_hash': hashlib.sha256(message.get('message-id').encode('utf-8')).hexdigest()[:8], + 'from': email.utils.parseaddr(message.get('from')), + 'to': email.utils.parseaddr(message.get('to')), + 'in_reply_to': message.get('in-reply-to'), + 'sent_at': email.utils.parsedate_to_datetime(message.get('date')).astimezone(pytz.utc).strftime('%Y-%m-%d %H:%M:%S'), + 'received_at': email.utils.parsedate_to_datetime(message.get_all('received')[0].rsplit(";")[-1]).astimezone(pytz.utc).strftime('%Y-%m-%d %H:%M:%S'), + 'subject': message.get('subject'), + 'body': message.get_body(('plain',)).get_content() + } + + emails.append(email_data) + + if email_data['from'] not in participants: + participants.append(email_data['from']) + + emails.sort(key = lambda email: email['received_at']) + + return template('mailing_list/emails_thread.html', thread=thread_data, emails=emails, + participants=participants, list_address=list_address, + repository=repository) diff --git a/web.service b/web.service new file mode 100644 index 0000000..78ae620 --- /dev/null +++ b/web.service @@ -0,0 +1,13 @@ +[Unit] +Description=Gunicorn instance to serve CLIF +After=network.target + +[Service] +User=git +Group=git +WorkingDirectory=/home/git/clif +ExecStart=/home/git/clif/venv/bin/gunicorn --workers 4 --bind localhost:5000 web:application +Restart=always + +[Install] +WantedBy=multi-user.target