[ themattreid ] » Scripts

CryptR, a quick and fun bash script to handle AES-256 encryption

admin — Tue, 24 Apr 2012 01:18:02 +0000

If you’ve ever wanted an easy way to encrypt and decrypt files on the command line without having to memorize or look up OpenSSL commands, then here’s a quick and easy script that provides that functionality. This script also functions as a reference to using getopt to process command line arguments. It should be noted that this has been functionally tested on Linux and OSX.

#!/bin/sh
# Date: 2011-07-29, update: 2012-04-23
# Author: Matt Reid
# Function: Decrypts and Encrypts files

function generate_digests() {
    echo "  Input file: $filein"
    openssl dgst -md5 $1
    echo "  Output file: $fileout"
    openssl dgst -md5 $2
}

function header() {
    echo " -------------------------------- "
    echo "|CryptR | security for the masses|"
    echo " -------------------------------- "
    echo "m.reid 2012.04.23 ver 2.68        "
    echo ""
}

function help() {
    echo "Purpose: Encrypts and Decrypts files via AES-256"
    echo "  -e, --encrypt       encrypt the file"
    echo "  -d, --decrypt       decrypt the file"
    echo "  -i, --input         input file to read"
    echo "  -o, --output        output file to write"
    echo ""
}

## Start GetOpt stuff
encrypt="no"    #encrypt function state (e,encrypt)
decrypt="no"    #decrypt function state (d,decrypt)
filein="no"   #filename IN flag (i,input)
fileout="no"  #filename OUT flag (o,output)

while [ $# -gt 0 ]; do
    case $1 in
        -e|--encrypt) encrypt="yes" ;;
        -d|--decrypt) decrypt="yes" ;;
        #long opts need additional shift
        -i|--input) filein="$2" ; shift;;
        -o|--output) fileout="$2" ; shift;;
        (--) shift; break;;
        (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;;
        (*) break;;
    esac
    shift
done
## End GetOpt

## Secure delete process
function sdelete() {
    if [ $(uname -s) == 'Darwin' ]; then
        srm $1
    elif [ $(uname -s) == 'Linux' ]; then
        shred -u $1
    fi
}

## Decrypt Process
function decrypt() {
    filein="$1"
    fileout="$2"
    if [ "$filein" = "no" ]; then
        echo -n "No encrypted file specified, what file are we decrypting: "
        read filein
    fi

    if [ -r "$filein" ]; then
        if [ "$fileout" = "no" ]; then
            fileout="$filein.decrypted"
        fi
        openssl enc -d -aes256 -in $filein -out $fileout
        generate_digests $filein $fileout
        exit 0;
    else
        echo "File '$filein' is not readable or does not exist."
        exit 1;
    fi
}

## Encrypt Process
function encrypt() {
    filein="$1"
    fileout="$2"
    if [ "$filein" = "no" ]; then
        echo -n "No input file specified, what file are we encrypting: "
        read filein
    fi

    if [ -r "$filein" ]; then
        if [ "$fileout" = "no" ]; then
            fileout="$filein.aes256"
        fi
        if [ -f "$fileout" ]; then
            echo "Output file exists already, encrypting will overwrite this file."
            echo -n "Do you want to encrypt anyway? [Y/n]: "
            read choice
            if [ "$choice" = "Y" ] || [ "$choice" = "y" ] || [ "$choice" = "" ]; then
                openssl enc -aes256 -in $filein -out $fileout
                generate_digests $filein $fileout
                sdelete $filein
                exit 0;
            else
                exit 2;
            fi
        else
            openssl enc -aes256 -in $filein -out $fileout
            generate_digests $filein $fileout
            sdelete $filein
            exit 0;
        fi
    else
        echo "Input file does not exist or is not readable. You're attempting to encrypt file: '$filein'"
        exit 1;
    fi
}

if [ "$encrypt" = "yes" ] && [ "$decrypt" = "no" ]; then
    encrypt $filein $fileout
elif [ "$decrypt" = "yes" ] && [ "$encrypt" = "no" ]; then
    decrypt $filein $fileout
else
    #clear
    header
    help
fi

Fun with Bash :: one liners

admin — Tue, 10 Apr 2012 18:13:30 +0000

Here are some quick and easy bash commands to solve every day problems I run into. Comment and leave some of your own if you like. I might update this post with new ones over time. These are just some common ones.

Iterate through directory listing and remove the file extension from each file
ls -1 | while read each; do new=`echo $each |sed 's/\(.*\)\..*/\1/'` && echo $new && mv "$each" "$new"; done

Output relevant process info, and nothing else
ps axo "user,pid,ppid,%cpu,%mem,tty,stime,state,command"| grep -v "grep" | grep $your-string-here

Setup a SOCKS5 proxy on localhost port 5050, to tunnel all traffic through a destination server
ssh -N -D 5050 username@destination_server'

Setup a SOCKS5 proxy via a remote TOR connection, using local port 5050 and remote TOR port 9050
ssh -L 5050:127.0.0.1:9050 username@destination_server'

Display text or code file contents to screen but don't display any # comment lines
sed -e '/^#/d' $1 < $file_name_here

Same as above but replacing # lines with blank lines
sed -e '/^#/g' $1 < $file_name_here

Find all symlinks in the current directory and subdirs
find ./ -type l -exec ls -l {} \;

Find all executable files in current directory and subdirs
find ./ -type f -perm -o+rx -exec ls -ld '{}' \;

Remove all files matching the input string
echo -n "filename match to remove [rm -i]: " && read f; find ./ -name ${f} -exec rm -i {} \;

Display largest ten files in current dir and subdirs
du -a ./ | sort -n -r | head -n 10

Display all files in current dir and subdirs in order of filesize
du -a ./ | sort -n -r

Generate a MD5 hash for the input string (not file)
Linux: echo -n "str: " && read x && echo -n "$x" | md5sum
OSX: echo -n "str: " && read x && echo -n "$x" | md5

Display a summary of all files in current and subdirs
for t in files links directories; do echo `find . -type ${t:0:1} | wc -l` $t; done 2> /dev/null

Download a website's SSL Certificate to a file for later use
openssl s_client -showcerts -connect $HOST:443 > $FILE-NAME.txt

MySQL Load Testing Framework – initial release

admin — Thu, 19 May 2011 17:52:48 +0000

It seems that everyone loves load testing these days. Problem is that everyone is using their own quick scripts, simple or complex, to drive their tests without the ability for other DBAs to duplicate those tests. Let’s say I write a great test and share my results and graphs on the blog – you want to run the same tests to see how your new DB servers compare in performance: this framework allows you to do that without duplicating any work or writing code. This is a basic release that will get the ball rolling. I’ve included some sample tests in the README file, so give them a try.

This codebase offers a user friendly framework for creating and visualizing MySQL database load test jobs. It is based around Sysbench, which is generally considered the industry standard load test application. The framework allows you to do the following:

Download it now: http://code.google.com/p/quadrant-framework/

Fun with Bash: aliases make your live easier… share your favorites

admin — Thu, 10 Feb 2011 04:18:40 +0000

I’ve always been a big fan of having a customized .bashrc file. The one I distribute to all of my servers has aliases for quick commands to save me time on the command line, functions that get work done when aliases are too simplistic, reporting for the server for each cli login, and of course a formatted and colored prompt (for terms that support colors). I also change certain aspects and commands based on the operating system since I’m not always on a redhat box or linux at all. Here’s my bashrc file – maybe you have some fun additions that you’d like to share. What saves you time on the command line?

Python for Automation: using pdsh for a menu-driven command execution environment

admin — Fri, 04 Feb 2011 22:15:56 +0000

I’ve been playing around with some quick system automation scripts that are handy to use when you don’t want / need to setup a chef or puppet action. I like to keep all of my hostnames and login details in a MySQL database (a cmdb actually) but for this example we’ll just use a couple of nested lists. This script executes commands in parallel across the hosts you choose in the menu system via the “pdsh” command, so make sure you have that installed before running. Alternately you can change the command call to use ssh instead of pdsh for a serialized execution, but that’s not as fun or fast. With some customizations here and there you can expand this to operate parallelized jobs for simplifying daily work in database administration, usage reporting, log file parsing, or other system automation as you see fit. Here’s the code. Comments welcome as always!

#!/usr/bin/env python
## NAME: menu_parallel_execution.py
## DATE: 2011-02-04
## AUTHOR: Matt Reid
## WEBSITE: http://kontrollsoft.com
## EMAIL: mreid@kontrollsoft.com
## LICENSE: BSD http://www.opensource.org/licenses/bsd-license.php

import commands
import sys
import os
import operator

ver = sys.version.split(' ')[0].split(".")
major=ver[:1]
minor=ver[1:2]
version="%s.%s"%(major[0],minor[0])
if version in ('2.4','2.3','2.2','2.1','2.0'):
	pyver = "old"
else:
	from subprocess import Popen, PIPE, STDOUT, call
	pyver = "new"

def main():
        #This section of host definitions could be replaced by a database
        #call if you are storing your hosts in that manner, but we'll keep
        #this script simple for the time being.
	dc0 = [
		["host0-0","10.0.0.1","ssh-username"],
		["host0-1","10.0.0.2","ss-username"]]

	dc1 = [
		["host1-0","10.1.0.1","ssh-username"],
		["host1-1","10.1.0.2","ss-username"]]

	dc2 = [
		["host2-0","10.2.0.1","ssh-username"],
		["host2-1","10.2.0.2","ss-username"]]

	print '''[1] Datacenter-1
[2] Datacenter-2
[3] Datacenter-3
'''
	dc = int(raw_input("Datacenter ID: "))
	if dc == 1:
		hosts = dc0
	elif dc == 2:
		hosts = dc1
	elif dc == 3:
		hosts = dc3

	nodes = []
	stay = True
	while stay == True:
		i = 0
		nest = sorted(hosts, key=operator.itemgetter(0))
		for x in range(len(nest)):
			print "[%i] %s | %s"%(i,nest[i][1],nest[i][0])
			i+=1

		print "\nSelect node to add to execution list"
		ch = int(raw_input("ID: "))
		xx = [nest[ch][1],nest[ch][2]]
		nodes.append(xx)
		s = str(raw_input("\nAdd another node? [Y/n] "))
		if s == "n" or s == "N":
			stay = False

	if(pyver == "new"):
		addrs = ""
		for node in nodes:
			address = node[1]+"@"+node[0]+","
			address = address.rstrip("\n")
			addrs = addrs+address

		addrs = addrs.strip(",")
		cmd = str(raw_input("\nEnter the command to execute: "))				

		try:
			c = "pdsh -w %s %s"%(addrs,cmd)
			print "Executing: %s"%(c)
			call(c,shell=True)
		except:
			print "Failed to execute pdsh command: %s"%(c)
			sys.exit(1)

		sys.exit(0)

	if(pyver == "old"):
		print "Please upgrade to Python 2.6+"
		sys.exit(1)

## START
if __name__ == "__main__":
	try:
		retval = main()
	except (KeyboardInterrupt, SystemExit):
		sys.exit(1)

Simple Python: a job queue with threading

admin — Fri, 21 Jan 2011 02:18:08 +0000

Every so often you need to use a queue to manage operations in an application. Python makes this very simple. Python also, as I’ve written about before, makes threading very easy to work with. So in this quick program I’ll describe via comments, how to make a simple queue where each job is processed by a thread. Integrating this code to read jobs from a mysql database would be trivial as well; simply replace the “jobs = [..." code with a database call to a row select query.

#!/usr/bin/env python
## DATE: 2011-01-20
## FILE: queue.py
## AUTHOR: Matt Reid
## WEBSITE: http://themattreid.com
from Queue import *
from threading import Thread, Lock

'''this function will process the items in the queue, in serial'''
def processor():
    if queue.empty() == True:
        print "the Queue is empty!"
        sys.exit(1)
    try:
        job = queue.get()
        print "I'm operating on job item: %s"%(job)
        queue.task_done()
    except:
        print "Failed to operate on job"

'''set variables'''
queue = Queue()
threads = 4

'''a list of job items. you would want this to be more advanced,
like reading from a file or database'''
jobs = [ "job1", "job2", "job3" ]

”’iterate over jobs and put each into the queue in sequence”’
for job in jobs:
     print “inserting job into the queue: %s”%(job)
     queue.put(job)

”’start some threads, each one will process one job from the queue”’
for i in range(threads):
     th = Thread(target=processor)
     th.setDaemon(True)
     th.start()

”’wait until all jobs are processed before quitting”’
queue.join()

A simple load test script in Python

admin — Wed, 03 Nov 2010 20:14:46 +0000

Lately I’ve had to do some environment load testing so I wrote this quick script. It can be modified as needed but the basic idea is that it spawns $x threads (–threads) and then sends two connections (or however many you want with –per-connection=) per thread to the URL (–url=). You can have it wait a configurable time between connections as well (–wait=).

The url is appended with a 32 character randomized string so that any database/caching on the backend of the site isn’t serving data from a warm cache. You can hunt down the string length for 32 and change it to whatever you want. Feel free to change and use as needed, just keep my info at top.

#!/usr/bin/python
################################################################################
## DATE: 2010-10-26
## AUTHOR: Matt Reid
## MAIL: mreid@kontrollsoft.com
## SITE: http://kontrollsoft.com
## LICENSE: BSD http://www.opensource.org/licenses/bsd-license.php
################################################################################

from __future__ import division
import threading
import sys
import urllib2
import select
import random
import string
import getopt
import time

class threader(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
    def run(self):
        global url
        global per
        global u
        for i in range(per):
            if wait > 0:
                time.sleep(wait)
            str = randstr(32)
            # IMPORTANT: this is where we append the search string to the main URL
            # you might need to change this for your site.
            url = "%s/search/%s"%(u,str)
            print "polling url: %s"%(url)
            urllib2.urlopen(url)

def randstr(length):
    global url
    twoletters = [c+d for c in string.letters for d in string.letters]
    r = random.random
    n = len(twoletters)
    l2 = length//2
    lst = [None] * l2
    for i in xrange(l2):
        lst[i] = twoletters[int(r() * n)]
        if length & 1:
            lst.append(random.choice(string.letters))

    return "".join(lst)

def init_thread():
    backgrounds = []
    for thread in range(threads):
        print "Spawning thread: %s"%(thread)
        background = threader()
        background.start()
        backgrounds.append(background)
    for background in backgrounds:
        background.join()

def print_help():
    print '''loader.py - URL load test script
==================================================
Date: 2010-08-26
Website: http://themattreid.com
Author: Matt Reid
Email: themattreid@gmail.com
License: new BSD license
==================================================
Use the following flags to change default behavior

   Option                 Description
   --url=                 URL to test
   --per-connection=      Number of sequential reqests per connection (default 2)
   --threads=             Number of threads for url connections (default 50)
   --wait=                Time to wait in-between requests
   --help                 Print this message

   -u                     Same as --url
   -p                     Same as --per-connection
   -t                     Same as --threads
   -w                     Same as --wait
   -h                     Same as --help
   '''

def main():
    init_thread()
    sys.exit(0)

if __name__ == "__main__":
    global threads #num threads/connections to open
    global u #url to hit
    global per #per connection url hits
    try:
        options, remainder = getopt.getopt(
            sys.argv[1:], 'ptuw', ['per-connection=',
                                   'threads=',
                                   'url=',
                                   'wait=',
                                   'help'])
    except getopt.GetoptError, err:
        print str(err)
        sys.exit(2)

    for opt, arg in options:
        if opt in ('--per-connection'):
            per = int(arg)
        elif opt in ('--threads'):
            threads = int(arg)
        elif opt in ('--url'):
            u = arg
        elif opt in ('--wait'):
            wait = int(arg)
        elif opt in ('--help'):
            print_help()
            sys.exit(2)

    try:
        threads
    except NameError:
        print "No thread quantity specified."
        print_help()
        sys.exit(2)
    try:
        per
    except NameError:
        per = 2
    try:
        u
    except NameError:
        print "No URL Specified"
        print_help()
        sys.exit(2)
    try:
        wait
    except NameError:
        wait=0

    main()

Easy Python: multi-threading MySQL queries

admin — Mon, 30 Aug 2010 21:04:24 +0000

There are many times when writing an application that single threaded database operations are simply too slow. In these cases it’s a matter of course that you’ll use multi-threading or forking to spawn secondary processes to handle the database actions. In this simple example for Python multi-threading you’ll see the how simple it is to improve the performance of your python app.

#!/usr/bin/python
## DATE: 2010-08-30
## AUTHOR: Matt Reid
## WEBSITE: http://themattreid.com
## LICENSE: BSD http://www.opensource.org/licenses/bsd-license.php
## Copyright 2010-present Matt Reid

from __future__ import division
from socket import gethostname;
import threading
import sys
import os
import MySQLdb

class threader(threading.Thread):
    def __init__(self,method):
        threading.Thread.__init__(self)
        self.tx =
        self.method = method
    def run(self):
        run_insert()

def run_insert():
    sql = "INSERT INTO table (`id`,`A`,`B`,`C`) VALUES (NULL,'0','0','0');")
        try:
            cursor.execute(sql)
            db.commit()
        except:
            print "insert failed"

def init_thread(): backgrounds = []
    for db in connections:
       logger("Spawning thread: %s"%(db),"d")
       quant = tx / THREADS
       background = threader(method,quant,db)        
       background.start()
       backgrounds.append(background)
    for background in backgrounds:
       background.join()

def main():
    try:
        init_thread()
    except:
        print "failed to initiate threads"

    sys.exit(0)

if __name__ == "__main__":
    mysql_host = "localhost" #default localhost
    mysql_pass = "pass" #default dbbench
    mysql_user = "user" #default dbbench
    mysql_port = 3306 #default 3306
    mysql_db = "schema" #default dbbench
    threads = 4 #must be INT not STR #create connection pool

    connections = []
    for thread in range(THREADS):
      try:
       connections.append(MySQLdb.connect(host=mysql_host, user=mysql_user, passwd=mysql_pass, db=mysql_db, port=mysql_port))
      except MySQLdb.Error, e:
       print "Error %d: %s"%(e.args[0], e.args[1])
       sys.exit (1)

    main()

Easy Python: display LVM details in XML

admin — Sun, 15 Aug 2010 22:35:30 +0000

If you need to work with LVM in your scripts but haven’t found a good method to access details about Logical Volume Groups, here’s a simple Python script that will print the details about any volumes on your system. This could be useful for writing a partition check script for your MySQL data directory (if you’re not using a standard monitoring system like Nagios).

import sys
import os
import commands
import subprocess
import select

def lvm():
    print ""
    LVM_PATH = "/sbin"
    LVM_BIN = os.path.join(LVM_PATH, 'lvm')
    argv = list()
    argv.append(LVM_BIN)
    argv.append("lvs")
    argv.append("--nosuffix")
    argv.append("--noheadings")
    argv.append("--units")
    argv.append("b")
    argv.append("--separator")
    argv.append(";")
    argv.append("-o")
    argv.append("lv_name,vg_name,lv_size")

    process = subprocess.Popen(argv, stdout=subprocess.PIPE)
    output = ""
    out = process.stdout.readline()
    output += out
    lines = output.splitlines()
    for line in lines:
        line = line.strip()
        words = line.split(";")

        lvname = words[0].strip()
        vgname = words[1].strip()
        lv_size = int(words[2])
        print '''
    %s
    %s
    %s
  '''%(lvname, vgname, lv_size)

    print ""

lvm()

Easy MySQL: how to backup databases to a remote machine

admin — Fri, 13 Aug 2010 21:27:39 +0000

Here’s a simple answer to a simple question. “How do I run a backup of MySQL to another machine without writing to the local server’s filesystem?” – this is especially useful if you are running out of space on the local server and cannot write a temporary file to the filesystem during backups.

Method one – this writes a remote file.
mysqldump [options] [db_name|--all-databases]| gzip -c | ssh user@host.com "cat > /path/to/new/file.sql.gz"

Method two – this writes directly into a remote mysql server
mysqldump [options] [db_name|--all-databases]| mysql --host=[remote host] –user=root –password=[pass] [db_name]