Python Standard Library

Subprocess

The subprocess module allows you to spawn new processes, connect to their input/output/error pipes, and obtain their return codes. This module intends to replace several older modules and functions:

os.system
os.spawn*
os.popen*
popen2.*
commands.*

http://sharats.me/the-ever-useful-and-neat-subprocess-module.html
https://pymotw.com/2/subprocess/

Get output from command

ls_output = subprocess.check_output(['ls', '-l'])

Note: this will not work

subprocess.check_output(['ls', '|', 'wc', '-l'])

Getting the return code (Running via shell, without output)

The default value for shell is False

subprocess.call('ls | wc -l', shell=True)
subprocess.call(['ls', '-l'], shell=True)
subprocess.call(['ls', '-l'])

Note 1: this will not working

subprocess.call('ls -l')

Popen class

Watching both stdout and stderr

proc = Popen('svn co svn+ssh://myrepo', stdout=PIPE)
for line in proc.stdout:
    print line

from subprocess import Popen, PIPE
from threading import Thread
from Queue import Queue, Empty

io_q = Queue()

def stream_watcher(identifier, stream):

    for line in stream:
        io_q.put((identifier, line))

    if not stream.closed:
        stream.close()

proc = Popen('svn co svn+ssh://myrepo', stdout=PIPE, stderr=PIPE)

Thread(target=stream_watcher, name='stdout-watcher',
        args=('STDOUT', proc.stdout)).start()
Thread(target=stream_watcher, name='stderr-watcher',
        args=('STDERR', proc.stderr)).start()

def printer():
    while True:
        try:
            # Block for 1 second.
            item = io_q.get(True, 1)
        except Empty:
            # No output in either streams for a second. Are we done?
            if proc.poll() is not None:
                break
        else:
            identifier, line = item
            print identifier + ':', line

Thread(target=printer, name='printer').start()

Passing an environment

The env argument to Popen (and others) lets you customize the environment of the command being run. If it is not set, or is set to None, the current process's environment is used, just as documented.

p = Popen('command', env=dict(os.environ, my_env_prop='value'))

Execute in a different directory

subprocess.call('./ls', cwd='/bin')

Multiprocessing

Python OS Library

os.listdir
os.path
- os.path.exists
os.remove
os.mkdir

import os
basedir = os.path.abspath(os.path.dirname(__file__))

def generate_zip(function_name):

    # wipe function folder if exists
    dir_path = '_deploy/{}'.format(function_name)
    if os.path.exists(dir_path):
        shutil.rmtree(dir_path)
    else:
        os.mkdir(dir_path)

    # copy code, modules, etc to directory
    copy_tree('lambda/src/{}'.format(function_name), dir_path)

    zip_path = '_deploy/_zip/{}.zip'.format(function_name)
    if os.path.exists(zip_path):
        os.remove(zip_path)

    # zip contents of dir
    shutil.make_archive('_deploy/_zip/{}'.format(function_name), 'zip', dir_path)

shutil

shutil.rmtree
shutil.make_archive

Argparse

import argparse
parser = argparse.ArgumentParser()
parser.add_argument("profile_name")
parser.add_argument("bucket_name")
parser.add_argument("lambda_function_arn")
args = parser.parse_args()

ConfigParser

>>> from configparser import ConfigParser
>>> cfg = ConfigParser()
>>> cfg.read('config.ini')
['config.ini']
>>> cfg.sections()
['installation', 'debug', 'server']
>>> cfg.get('installation','library')
'/usr/local/lib'
>>> cfg.getboolean('debug','log_errors')
True
>>> cfg.getint('server','port')
8080
>>> cfg.getint('server','nworkers')
32
>>> print(cfg.get('server','signature'))

Logging

Logging from multiple modules
https://docs.python.org/3/howto/logging.html#logging-from-multiple-modules

import logging

# create logger
logger = logging.getLogger('simple_example')
logger.setLevel(logging.DEBUG)

# create console handler and set level to debug
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)

# create formatter
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

# add formatter to ch
ch.setFormatter(formatter)

# add ch to logger
logger.addHandler(ch)

# 'application' code
logger.debug('debug message')
logger.info('info message')
logger.warn('warn message')
logger.error('error message')
logger.critical('critical message')

Output is:

$ python simple_logging_module.py
2005-03-19 15:10:26,618 - simple_example - DEBUG - debug message
2005-03-19 15:10:26,620 - simple_example - INFO - info message
2005-03-19 15:10:26,695 - simple_example - WARNING - warn message
2005-03-19 15:10:26,697 - simple_example - ERROR - error message
2005-03-19 15:10:26,773 - simple_example - CRITICAL - critical message

import logging
import logging.config

logging.config.fileConfig('logging.conf')

# create logger
logger = logging.getLogger('simpleExample')

# 'application' code
logger.debug('debug message')
logger.info('info message')
logger.warn('warn message')
logger.error('error message')
logger.critical('critical message')

[loggers]
keys=root,simpleExample

[handlers]
keys=consoleHandler

[formatters]
keys=simpleFormatter

[logger_root]
level=DEBUG
handlers=consoleHandler

[logger_simpleExample]
level=DEBUG
handlers=consoleHandler
qualname=simpleExample
propagate=0

[handler_consoleHandler]
class=StreamHandler
level=DEBUG
formatter=simpleFormatter
args=(sys.stdout,)

[formatter_simpleFormatter]
format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
datefmt=