adding information to the readme

README.md

@@ -4,6 +4,44 @@ Crunch is a python based command line utility that allows for manual rebuilding
 index pages, error pages and the home page. It also supports parsing email to create new 
 posts. It was created to run amdavidson.com.
 
+Usage (from `crunch.py --help`):
+
+    usage: crunch.py [-h] [--all] [--clean] [--dependencies] [--email] [--error]
+                     [--extras] [--feed] [--galleries] [--home] [--indexes]
+                     [--new] [--no-http] [--pages] [--posts] [--serve] [--setup]
+                     [--single SINGLE] [--verbose]
+
+    optional arguments:
+      -h, --help       show this help message and exit
+      --all            Builds the entire site.
+      --clean          Empties the build folder.
+      --dependencies   Builds all the dependencies, ignored unless used with
+                       --single, --new, or --email.
+      --email          Reads an email message from STDIN and parses to create a
+                       new post. Overrides --all, --posts, --indexes, --home, and
+                       --single
+      --error          Generates static error pages.
+      --extras         Generates minified css and js files.
+      --feed           Generates RSS feed.
+      --galleries      Generates galleries.
+      --home           Builds the home page.
+      --indexes        Builds the index pages.
+      --new            Starts an interactive sesson to create a new post. *Not yet
+                       implemented*
+      --no-http        Prevents crunch from contacting external sources during the
+                       build.
+      --pages          Builds all static pages.
+      --posts          Builds all posts.
+      --serve          Starts a lightweight HTTP server to serve build folder to
+                       localhost. Not intended for production use.
+      --setup          Creates a basic blog framework to start with. *Not yet
+                       implemented.*
+      --single SINGLE  Builds a single post. Takes a filename as an argument or
+                       use - to read from STDIN. Overrides all other build instructions.
+                       *Not yet implemented.*
+      --verbose        Enables information display other than errors.
+
+
 The configuration is stored in a file called conf.yaml in the same directory as crunch.
 
 An example configuration follows:
@@ -30,6 +68,8 @@ An example configuration follows:
     public_folder: public
     images_folder: images
     galleries_folder: galleries
+    css_folder: css
+    scripts_folder: scripts
     home_count: 5
     image_width: 640
     image_height: 640

crunch.py

@@ -29,7 +29,7 @@ except:
   yaml_available = False
 
 try:
-  import markdown
+  import markdown2
   markdown_available = True
 except: 
   markdown_available = False
@@ -58,6 +58,8 @@ if argparse_available:
                       post. Overrides --all, --posts, --indexes, --home, and --single')
   parser.add_argument('--error', dest='error', action='store_true', 
                       help='Generates static error pages.')
+  parser.add_argument('--extras', dest='extras', action='store_true',
+                      help='Generates minified css and js files.')
   parser.add_argument('--feed', dest='feed', action='store_true',
                       help='Generates RSS feed.')
   parser.add_argument('--galleries', dest='galleries', action='store_true',
@@ -120,8 +122,8 @@ posts_folder = base_folder + '/' + conf['posts_folder']
 public_folder = base_folder + '/' + conf['public_folder']
 images_folder = base_folder + '/' + conf['images_folder']
 galleries_folder = base_folder + '/' + conf['galleries_folder']
-
+css_folder = base_folder + '/' + conf['css_folder']
-
+scripts_folder = base_folder + '/' + conf['scripts_folder']
 
 
 ### Classes
@@ -215,7 +217,7 @@ class Post:
     # if markdown is available, use that to process the post body.
     self.markdown = body
     if markdown_available:
-      self.content = markdown.markdown(str(body))
+      self.content = markdown2.markdown(str(body), extras=["code-color", "code-friendly"])
     else:
       if args.verbose: print 'WARN: markdown unavailable, using raw post data.'
       self.content = self.markdown
@@ -286,10 +288,7 @@ def format_layout(page):
 
     <link rel="icon" type="image/png" href="/images/favicon.png" />
 
-    <link rel="stylesheet" type="text/css" href="/css/base.css" />
+    <link rel="stylesheet" type="text/css" href="/css/app.css" />
-    <link rel="stylesheet" type="text/css" href="/css/skeleton.css" />
-    <link rel="stylesheet" type="text/css" href="/css/layout.css" />
-    <link rel="stylesheet" type="text/css" href="http://fonts.googleapis.com/css?family=Muli:300,300italic|PT+Sans+Narrow:700|Anonymous+Pro" />
 
     <link rel="alternate" type="application/atom+xml" title="amdavidson.com feed" 
           href="/index.xml" />
@@ -307,23 +306,25 @@ def format_layout(page):
       %(body)s
     </div>
     <div class="four columns">
-      <h6>About</h6>
+      <h6><a href="/about.htm">about</a></h6>
       <p class="small">amdavidson.com is a simple blog run by Andrew Davidson, a 
       manufacturing engineer with a blogging habit. He sometimes posts 140 character 
       <a href="http://twitter.com/amdavidson">tidbits</a>, shares 
       <a href="/">photos</a>, and saves 
       <a href="http://pinboard.in/u:amdavidson/">links</a>. You can also see posts 
-      dating <a href="/archives">back to 2005</a>.</p>
+      dating <a href="/archives.htm">back to 2005</a>.</p>
       
       <div id="twitter" style="display:none;">
-        <h6><a href="http://twitter.com/amdavidson">Twitter</a></h6>
+        <h6><a href="http://twitter.com/amdavidson">tweeted</a></h6>
-        <p class="small"><span id="tweet"></span></p>
+        <p class="small"><span id="tweet"></span><br/>
+        <span id="tweet-date"></span></p>
       </div>
 
       <div id="pinboard" style="display:none;">
-        <h6><a href="http://pinboard.in/u:amdavidson">Pinboard</a></h6>
+        <h6><a href="http://pinboard.in/u:amdavidson">bookmarked</a></h6>
         <p class="small"><a id="pin-link" href="/"><span id="pin-title"></span></a><br/>
-        <span id="pin-description"></span></p>
+        <span id="pin-description"></span><br/>
+        <span id="pin-date"></span></p>
       </div>
 
       <h6>Search</h6>
@@ -339,19 +340,10 @@ def format_layout(page):
       </form>
                         
     </div>
-    
+    </div>
-    <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js">
-      </script>
-    <script src="/scripts/jquery.timeago.js"></script>
-    <script src="/scripts/twitter.js"></script>
-    <script src="/scripts/pinboard.js"></script>
-    
-    <script type="text/javascript">
-      jQuery(document).ready(function() {
-          jQuery("span.timeago").timeago();
-      });
-    </script>
 
+    <script src="/scripts/zepto.min.js"></script>
+    <script src="/scripts/app.js"></script>
     <script src="http://mint.amdavidson.com/?js" type="text/javascript"></script>
 
   </body>
@@ -434,20 +426,26 @@ def format_gallery_single(image):
       <div class="eleven columns">
         <h3>%(name)s</h3>
         <p style="text-align:center;"><a href="%(full_url)s">
-          <img src="%(mid_url)s" />
+          <img class="scale-with-grid" src="%(mid_url)s" />
         </a></p>
       </div>
   """ % { 'name': image.name(), 'full_url': image.full_url(), 'mid_url': image.mid_url() }
   
 def format_gallery_thumb(image):
   return """
-      <a href="%(mid_page)s"><img src="%(thm_path)s" /></a>
+      <div class="thumbnail">
+        <a href="%(mid_page)s"><img src="%(thm_path)s" /></a>
+      </div>
   """ % { 'mid_page': image.mid_page(), 'thm_path': image.thumbnail_url() }
 
 ##########################################################################################
 ### Helper Functions
 ##########################################################################################
 
+
+# get_recent() takes in an integer that sets the number of recent posts to get, it 
+# returns a list of post objects in reverse chronological order. This function is used
+# in crunch_feed() and crunch_home().
 def get_recent(count):
   # Create an empty variable to store posts in.
   post_list = []
@@ -516,6 +514,8 @@ def ensure_build_folder():
     shutil.copytree(public_folder, build_folder)
     shutil.copytree(images_folder, build_folder + '/' + conf['images_folder'])
     os.mkdir(build_folder + '/' + conf['galleries_folder'])
+    os.mkdir(build_folder + '/' + conf['css_folder'])
+    os.mkdir(build_folder + '/' + conf['scripts_folder'])
     return 2
   return 1
 
@@ -545,14 +545,17 @@ def crunch_pages():
     if filename.endswith(conf['extension']):
       if args.verbose: print 'Building ' + filename
       
-      # Fire up a markdown object.
+      # Split the page header from the body.
-      md = markdown.Markdown(extensions = ['meta'])
+      header, body = open(pages_folder + '/' + filename).read().split('\n\n', 1)               
+  
+      # Pull a dict from the yaml in the header.
+      y = yaml.load(header)
       
       # Parse the post and grab the content.
-      content = md.convert(open(pages_folder + '/' + filename).read())
+      content = markdown2.markdown(body, extras=["code-color", "code-friendly"])
       
       # Pull the title out of the metadata.
-      title = md.Meta['title'][0]
+      title = y['title']
       
       # Generate the url
       url = '/' + filename.rstrip(conf['extension']) + '.htm'
@@ -602,28 +605,10 @@ def crunch_posts():
             # Only process files with the correct extension per `conf.yaml`.
             if i.endswith(conf['extension']):
               if args.verbose: print '\t\t' + i
-
-              # Create a new post object and parse the post file.
-              post = Post()
-              f = open(posts_folder + '/' + year + '/' + month + '/' + i)
-              post.filename = i
-              post.parse(f.read())        
-              f.close()
               
-              # Create a new page for this post.
+              # Process the post
-              page = Page()
+              crunch_single(open(posts_folder + '/' + year + '/' + month \
-              page.title = str(post.title) + ' | ' + page.title
+                + '/' + i).read())
-              page.body = post.formatted()
-
-              # Save the page to the $build/$year/$month folder.
-              n = open(build_folder + '/' + year + '/' + month + '/' + post.slug + \
-                       '.htm', "w")
-              n.writelines(page.formatted())
-              n.close
-              
-              os.chmod(build_folder + '/' + year + '/' + month + '/' + post.slug + \
-                       '.htm', 0644)
-
 
 # Function to process the home file.
 def crunch_home():
@@ -660,8 +645,8 @@ def crunch_indexes():
   if args.verbose: print 'Building the indexes.'
   
   # Start the body for the archives.htm page.
-  archives_body = '<div class="eleven-columns"><h3>Post Archives</h3>\n' + \
+  archives_body = '\t<div class="eleven-columns">\n\t\t<h3>Post Archives</h3>\n\t</div>\n' + \
-    '<ul id="acc" class="square">\n'
+    '\t<div class="eleven-columns">\n\t\t<ul class="square">\n'
 
   # Grab all the years in the posts folder.
   for year in sorted(os.listdir(posts_folder), reverse=True): 
@@ -671,8 +656,8 @@ def crunch_indexes():
       if args.verbose: print 'Building indexes for ' + year + ':'
 			
       # Add an entry to archives.htm
-      archives_body += '<li><a href="/' + year + '">' + year + '</a>\n\t\
+      archives_body += '\t\t\t<li><a href="/' + year + '">' + year + '</a>\n\t\
-        <ul class="circle">\n'
+        \t\t<ul class="circle">\n'
 
       # Make a corresponding year folder in the build folder if it doesn't exist.
       year_path = build_folder + '/' + year
@@ -689,7 +674,7 @@ def crunch_indexes():
           if args.verbose: print "\t" + month
 
           # Add an entry to archives.htm.
-          archives_body += '\t\t<li><a href="/' + year + '/' + month + '">' + month \
+          archives_body += '\t\t\t\t\t<li><a href="/' + year + '/' + month + '">' + month \
               + '</a>\n'
 
           # Make a corresponding month folder in the build folder if it doesn't exist.
@@ -730,11 +715,11 @@ def crunch_indexes():
           month_page.body = month_body
 
           # Write out the titles to the posts to archives.htm in ascending order.
-          archives_body += '\t\t\t<ul>\n'
+          archives_body += '\t\t\t\t\t\t<ul>\n'
           for post in sorted(month_catch, key=lambda post: post.time, reverse=True):
-            archives_body += '\t\t\t\t<li><a href="' + post.url() + '">' + \
+            archives_body += '\t\t\t\t\t\t\t<li><a href="' + post.url() + '">' + \
               str(post.title) + '</a></li>\n'
-          archives_body += '\t\t\t</ul></li>\n'
+          archives_body += '\t\t\t\t\t\t</ul>\n\t\t\t\t\t</li>\n'
 
           # Write out the month page into the build folder.
           m = open(build_folder + '/' + year + '/' + month + '/index.htm', "w")
@@ -743,7 +728,7 @@ def crunch_indexes():
           os.chmod(build_folder + '/' + year + '/' + month + '/index.htm', 0644)
       
       # Close out the list of months in archive.htm
-      archives_body += '\t</ul></li>\n'
+      archives_body += '\t\t\t\t</ul>\n\t\t\t</li>\n'
       
       # Once all the posts for the current year have been processed, make a new
       # Page object for the year.
@@ -764,7 +749,7 @@ def crunch_indexes():
       os.chmod(build_folder + '/' + year + '/index.htm', 0644)
     
   # Close out the list of years in archive.htm
-  archives_body += '</ul>\n</div>'
+  archives_body += '\t\t</ul>\n\t</div>'
   
   archives_page = Page()
   archives_page.title = 'Archives | ' + archives_page.title
@@ -857,15 +842,21 @@ def crunch_email(message):
             exif = dict(original._getexif().items())
           else: 
             exif = False 
-            
+          
+          
+          
           if not exif == False:
-            if args.verbose: print 'Image is rotated, correcting.'
+            try:
-            if exif[orientation] == 3:
+              if args.verbose: print 'Image is rotated, correcting.'
-              original = original.rotate(180, expand=True)
+              if exif[orientation] == 3:
-            elif exif[orientation] == 6:
+                original = original.rotate(180, expand=True)
-              original = original.rotate(270, expand=True)
+              elif exif[orientation] == 6:
-            elif exif[orientation] == 8:
+                original = original.rotate(270, expand=True)
-              original = original.rotate(90, expand=True)
+              elif exif[orientation] == 8:
+                original = original.rotate(90, expand=True)
+            except:
+              if args.verbose: print 'Cannot detect rotation from EXIF.'
+              
 
           # Create empty resized var.
           resized = False
@@ -973,6 +964,7 @@ def crunch_email(message):
     f.write('title: ' + title + '\n')
     f.write('date: ' + str(epoch_time) + '\n')
     f.write('author: ' + conf['author'] + '\n')
+    f.write('slug: ' + slug + '\n')
     if not short == None:
       f.write('short: ' + short + '\n')
     f.write('\n')
@@ -1194,6 +1186,7 @@ def crunch_gallery(name):
   # Make a destination gallery.
   if not os.path.exists(build_folder + '/' + conf['galleries_folder'] + '/' + name):
     os.mkdir(build_folder + '/' + conf['galleries_folder'] + '/' + name)
+    os.chmod(build_folder + '/' + conf['galleries_folder'] + '/' + name, 0755)
 
   images = ''
   
@@ -1201,12 +1194,19 @@ def crunch_gallery(name):
   for file in os.listdir(galleries_folder + '/' + name):
 
     # Process the meta data file.
-    if file == 'meta.md':
+    if file == 'meta.yaml':
       if args.verbose: print '\tProcessing metadata.'
-      md = markdown.Markdown(extensions = ['meta'])
+      
-      description = md.convert(open(galleries_folder + '/' + name + '/' + file, \
+      a = open(galleries_folder + '/' + name + '/' + file, \
-        'r').read())
+        'r').read().split('\n\n', 1)               
-  
+      
+      y = yaml.load(a[0])
+      
+      try:
+        description = '<div class="eleven columns">' + \
+          markdown2.markdown(str(a[1]), extras=["code-color", "code-friendly"])  
+      except:
+        description = '<div class="eleven columns">'
   
     # Copy all the images.
     if filter(file.endswith, image_extensions):
@@ -1231,15 +1231,19 @@ def crunch_gallery(name):
         os.chmod(build_folder + '/' + conf['galleries_folder'] + '/' + name + '/' + \
           i.name() + '.htm', 0644)
    
+   
+  images += "</div>"
+   
   gal_page = Page()
   
-  leader = '<h3>' + str(md.Meta['title'][0]) + '</h3>\n<p class="small">' + \
+  leader = '<div class="eleven columns">\n<h3>' + str(y['title']) + \
-    time.strftime("posted on %Y-%m-%d at %I:%M %p", 
+    '</h3>\n<p class="small">' + \
-      time.localtime(float(md.Meta['date'][0]))) + '</p>'
+    time.strftime("posted on %Y-%m-%d at %I:%M %p", \
+    time.localtime(float(y['date']))) + '</p></div>'
   
   gal_page.body = leader + description + images
   
-  gal_page.title = str(md.Meta['title'][0]) + ' | ' + gal_page.title
+  gal_page.title = str(y['title']) + ' | ' + gal_page.title
   
   f = open(build_folder + '/' + conf['galleries_folder'] + '/' + name + '/index.htm', 'w')
   f.writelines(gal_page.formatted())
@@ -1256,7 +1260,101 @@ def crunch_gallery_all():
   for dir in [x[0] for x in os.walk(galleries_folder)]:
     if not re.search(conf['galleries_folder'] + '$', dir):
       crunch_gallery(os.path.basename(dir))
+
       
+      
+      
+# Combine and minify CSS and JS.
+def crunch_extras():
+  if args.verbose: print 'Combining and minifying stylesheets and scripts.'
+  
+  # Make some empty variables to put the minified content in.
+  css_min = []
+  js_min = []
+  
+  # Iterate through the css files.
+  for file in sorted(os.listdir(css_folder)):
+    
+    # Ignore excluded files. 
+    if not file.startswith('_'):
+      
+      # Only Process all the non-minified CSS files.
+      if file.endswith('.css') and not file.endswith('.min.css'):
+      
+        # Read the file into a tmp var.
+        tmp = open(css_folder + '/' + file).read()
+        
+        # Kill all the comments.
+        tmp = re.sub( r'/\*[\s\S]*?\*/', '', tmp)
+        
+        # Minimize the whitespace.
+        tmp = ' '.join(tmp.split())
+        
+        # Add it to the new file.
+        css_min.append(tmp)
+      
+      # If the file is minified, we still want it but don't want to waste time.
+      if file.endswith('.min.css'):
+        css_min.append(open(css_folder + '/' + file).read())
+      
+    # If the file is excluded just copy it over.
+    if file.startswith('_'):
+      shutil.copy2(css_folder + '/' + file, build_folder + '/' + \
+        conf['css_folder'] + '/' + file.lstrip('_'))      
+  
+  # Write out our new minified CSS file.
+  f = open(build_folder + '/' + conf['css_folder'] + '/app.css', 'w')
+  f.writelines(''.join(css_min))
+  f.close
+  os.chmod(build_folder + '/' + conf['css_folder'] + '/app.css', 0644)      
+      
+  # Iterate through JS files.
+  for file in sorted(os.listdir(scripts_folder)):
+  
+    # Ignore excluded files.
+    if not file.startswith('_'):
+      
+      # Only bother with JS files and ignore pre-minified ones.
+      if file.endswith('.js') and not file.endswith('.min.js'):
+        # Read the file into a tmp var.
+        for line in open(scripts_folder + '/' + file).readlines():
+      
+          # Ignore comments lines.
+          if not re.match('//', line) and not re.match('\s+//', line):
+              
+              # minimize whitespace
+              line = ' '.join(line.split())          
+              
+              # add the minimized js to the new file
+              js_min.append(line)
+
+        # Kill all the comments.
+        #tmp = re.sub( r'\/\*.+?\*\/|\/\/.*(?=[\n\r])', "", tmp)
+        
+        # Minimize the whitespace. Can't eliminate as some is critical.
+        # Cannot be used unless comments are removed.
+        #tmp = re.sub(r'\s+', ' ', tmp)
+              
+        #js_min.append(tmp)
+      
+      # Included the minified js file, but don't process it.
+      if file.endswith('.min.js'):
+        js_min.append(open(scripts_folder + '/' + file).read())
+        
+    # Copy excluded files straight over with no changes.
+    if file.startswith('_'):
+      shutil.copy2(scripts_folder + '/' + file, build_folder + '/' + \
+        conf['scripts_folder'] + '/' + file.lstrip('_'))
+    
+  # Write out our new minified JS file.
+  f = open(build_folder + '/' + conf['scripts_folder'] + '/app.js', 'w')
+  f.writelines(''.join(js_min))
+  f.close
+  os.chmod(build_folder + '/' + conf['scripts_folder'] + '/app.js', 0644)
+
+    
+  
+  
   
 ##########################################################################################
 ### Party Time.
@@ -1296,7 +1394,7 @@ def main():
   
       # Re-process everything.
       if args.all:
-        if args.verbose: print 'Building it all.'
+        if args.verbose: print 'Building all the things.'
         # Make sure we have a build folder to use.
         ensure_build_folder()
         
@@ -1318,12 +1416,15 @@ def main():
         # Rebuild the feed.
         crunch_feed()
         
+        # Rebuild the extras.
+        crunch_extras()
+        
         # Build the galleries.
         crunch_gallery_all()
       
       # We're going to do a partial rebuild.
       elif args.posts or args.home or args.indexes or args.feed or args.galleries or \
-        args.pages:
+        args.pages or args.extras:
         
         ensure_build_folder()
         
@@ -1353,6 +1454,10 @@ def main():
         if args.feed:
           crunch_feed()
           
+        # Build the extras if the --extras flag is set.
+        if args.extras:
+          crunch_extras()
+          
         # Build the galleries if the --galleries flag is set.
         if args.galleries:
           crunch_gallery_all()
@@ -1398,7 +1503,7 @@ def main():
         server = SocketServer.TCPServer(("", conf['server_port']), handler)
       except:
         print "Port occupied... Retrying."
-        time.sleep(10)
+        time.sleep(5)
          
     # Change to the build folder. 
     os.chdir(build_folder)