Merge branch 'master' into fastmail

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+.pyc

Pipfile

@@ -6,6 +6,7 @@ name = "pypi"
 [packages]
 pygit2 = "*"
 requests = "*"
+pyyaml = "*"
 
 [dev-packages]
 

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "5f88aa6bcdda5527c39943563c5735df7d1f941270437f3a0518d31e4b830451"
+            "sha256": "b0181521bb016cf1b8bc7de817cc2f903eefd390e75c26e9d19618233e6548fd"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -103,6 +103,23 @@
             "index": "pypi",
             "version": "==1.1.1"
         },
+        "pyyaml": {
+            "hashes": [
+                "sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6",
+                "sha256:1cf708e2ac57f3aabc87405f04b86354f66799c8e62c28c5fc5f88b5521b2dbf",
+                "sha256:24521fa2890642614558b492b473bee0ac1f8057a7263156b02e8b14c88ce6f5",
+                "sha256:4fee71aa5bc6ed9d5f116327c04273e25ae31a3020386916905767ec4fc5317e",
+                "sha256:70024e02197337533eef7b85b068212420f950319cc8c580261963aefc75f811",
+                "sha256:74782fbd4d4f87ff04159e986886931456a1894c61229be9eaf4de6f6e44b99e",
+                "sha256:940532b111b1952befd7db542c370887a8611660d2b9becff75d39355303d82d",
+                "sha256:cb1f2f5e426dc9f07a7681419fe39cee823bb74f723f36f70399123f439e9b20",
+                "sha256:dbbb2379c19ed6042e8f11f2a2c66d39cceb8aeace421bfc29d085d93eda3689",
+                "sha256:e3a057b7a64f1222b56e47bcff5e4b94c4f61faac04c7c4ecb1985e18caa3994",
+                "sha256:e9f45bd5b92c7974e59bcd2dcc8631a6b6cc380a904725fce7bc08872e691615"
+            ],
+            "index": "pypi",
+            "version": "==5.3"
+        },
         "requests": {
             "hashes": [
                 "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee",

archivist.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+import sys, logging
+from archivist import Archivist
+
+command = sys.argv[1] if len(sys.argv) > 1 else "backup"
+source = sys.argv[2] if len(sys.argv) > 2 else "all"
+
+a = Archivist()
+a.run(command, source)
+

archivist/__init__.py

@@ -0,0 +1,10 @@
+import logging, sys
+
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+out_handler = logging.StreamHandler(sys.stdout)
+#out_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
+out_handler.setLevel(logging.INFO)
+log.addHandler(out_handler)
+
+from archivist.archivist import Archivist

archivist/archivist.py

@@ -0,0 +1,26 @@
+import logging, sys
+
+import archivist.pinboard as pinboard
+import archivist.github as github
+
+log = logging.getLogger(__name__)
+
+class Archivist():
+    def run(self, command, source):
+        if command == "backup":
+
+            if source == "github":
+                log.info("Backing up Github")
+                github.backup_github()
+            elif source == "pinboard":
+                log.info("Backing up Pinboard")
+                pinboard.backup_pinboard()
+            elif source == "all":
+                log.info("Backing up all sources")
+                github.backup_github()
+                pinboard.backup_pinboard()
+            else:
+                log.error("Source %s not implemented" % source)
+
+        else:
+            log.error("Command %s not implemented" % command)

archivist/github.py

@@ -1,16 +1,10 @@
-#!/usr/bin/env python3
-
-import requests
-import json
-import os
+import requests, json, logging, os
 from pathlib import Path
 import pygit2
-from lib import Log
 
 user = "amdavidson"
 backupdir = Path.home() / "backups/github"
-log = Log()
-log.LOGLEVEL = 1
+log = logging.getLogger(__name__)
 
 def backup_gh_repos():
     response = requests.get("https://api.github.com/users/"+user+"/repos")
@@ -18,14 +12,14 @@ def backup_gh_repos():
     repos = json.loads(response.text)
 
     for repo in repos:
-        log("Backing up: " + repo["name"])
+        log.info("Backing up: " + repo["name"])
         dest = backupdir / "repo" / repo["full_name"]
         localrepopath = pygit2.discover_repository(str(dest))
         if localrepopath == None:
-            log("Cloning repo...")
+            log.info("Cloning repo...")
             pygit2.clone_repository(repo['clone_url'], str(dest), bare=True)
         else:
-            log("Fetching updates...")
+            log.info("Fetching updates...")
             localrepo = pygit2.Repository(localrepopath).remotes["origin"].fetch()
 
 def backup_gh_gists():
@@ -34,18 +28,19 @@ def backup_gh_gists():
     gists = json.loads(response.text)
 
     for gist in gists:
-        log("Backing up: " + gist["id"])
+        log.info("Backing up: " + gist["id"])
         dest = backupdir / "gist" / gist["owner"]["login"] / gist["id"]
         localgistpath = pygit2.discover_repository(str(dest))
         if localgistpath == None:
-            log("Cloning gist...")
+            log.info("Cloning gist...")
             pygit2.clone_repository(gist['git_pull_url'], str(dest), bare=True)
         else:
-            log("Fetching updates...")
+            log.info("Fetching updates...")
             pygit2.Repository(localgistpath).remotes["origin"].fetch()
 
 def backup_github(): 
     backup_gh_repos()
     backup_gh_gists()
 
-backup_github()
+if __name__ == '__main__':
+    backup_github()

archivist/lib.py

@@ -1,5 +1,6 @@
-
+import yaml
 import sys 
+from pathlib import Path
 
 class Log():
     LOGLEVEL = 3
@@ -19,3 +20,17 @@ class Log():
                 print(logstring)
 
 
+class Config():
+
+    CONFIGFILE = Path.home() / ".archivist.yml"
+
+    def init(self):
+        with open(self.CONFIGFILE) as stream:
+            try:
+                return yaml.full_load(stream)
+            except yaml.YAMLError as exc:
+                print(exc)
+
+    
+    
+

archivist/pinboard.py

@@ -0,0 +1,40 @@
+import requests, json, datetime, os, time, glob, logging
+from pathlib import Path
+from archivist.lib import Config
+
+log = logging.getLogger(__name__)
+
+backupdir = Path.home() / "backups/pinboard"
+
+config = Config()
+c = config.init()
+
+def get_last_backup():
+    backups = backupdir.glob("*.json")
+    oldest = datetime.datetime.fromtimestamp(0)
+    for b in backups:
+        bdate = datetime.datetime.fromtimestamp(float(b.stem))
+        if bdate > oldest:
+            oldest = bdate
+    return oldest
+
+
+def backup_pinboard():
+    response = requests.get("https://api.pinboard.in/v1/posts/update?format=json&auth_token="+c["pinboard_user"]+":"+c["pinboard_token"])
+
+    apiupdated = datetime.datetime.strptime(json.loads(response.text)["update_time"], "%Y-%m-%dT%H:%M:%SZ")
+
+    if apiupdated > get_last_backup() :
+        log.info("New bookmarks added, pulling latest backup...")
+        response = requests.get("https://api.pinboard.in/v1/posts/all?format=json&auth_token="+c["pinboard_user"]+":"+c["pinboard_token"])
+        backupfile = str(time.time()) + ".json" 
+        backuppath = backupdir / backupfile
+        backupdir.mkdir(parents=True, exist_ok=True)
+
+        with open(backuppath, "w+") as f:
+            f.write(response.text)
+    else:
+        log.info("No new bookmarks since last backup.")
+
+if __name__ == '__main__':
+    backup_pinboard()

readme.md

@@ -1,5 +1,18 @@
-#Backup Scripts
+# Archivist 
 
-## Github
+The archivist is run from a either a central `archivist.py` script
+
+## Backing up all services
+
+Running `./archivist.py` alone will backup all configured services.
+
+## Backing up a single service
+
+Using Github as the example service to backup:
+
+Add `github_user` to `$HOME/.archivist.yml`
+
+### Centrally
+
+Run `./archivist.py backup github`
 
-Edit `backup-github.py` with your preferred options then call it to backup all public github repositories of a particular user.