#!/usr/bin/env python

"""Manta shell."""

import sys
import os
from posixpath import dirname as udirname, join as ujoin, \
    basename as ubasename, normpath as unormpath, \
    split as usplit
import datetime
import logging
import json
from pprint import pprint, pformat
import re
import time
import mimetypes
from hashlib import md5
from operator import itemgetter
import fnmatch
import subprocess
import webbrowser
import codecs
import optparse

# Use the local manta package if we're in the dev layout.
_dev_package_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, _dev_package_dir)
import manta
from manta import MantaError
from manta import cmdln
from manta import appdirs
if _dev_package_dir in sys.path:
    sys.path.remove(_dev_package_dir)
del _dev_package_dir



#---- Python version compat

try:
    from urllib.parse import urlparse # python3
except ImportError:
    from urlparse import urlparse # python2



#---- globals

log = logging.getLogger("mantash")

# Use a separate cache dir tree if running with a different UID (e.g. if
# acting as root via `sudo -E`). This avoids cache dir permission
# issues. See <https://github.com/joyent/python-manta/issues/20>.
CACHE_DIR = appdirs.user_cache_dir("mantash-%s" % os.geteuid(), "Joyent")
HTTP_CACHE_DIR = os.path.join(CACHE_DIR, "http")

USER_AGENT = "mantash/%s (%s) Python/%s" % (
    manta.__version__, sys.platform, sys.version.split(None, 1)[0])
KNOWN_USERS_PATH = os.path.join(CACHE_DIR, 'known-users.json')

DEFAULT_PS1 = r'[\m\w]$ '




#---- errors

class MantashError(Exception):
    pass

class MantaObjectDoesNotExist(MantashError):
    pass


#---- the shell


def optparser_setattr(**kwargs):
    """Set attrs on the function's `optparser` attribute.

    This is modelled off `cmdln.option()`."""
    def decorate(f):
        if not hasattr(f, "optparser"):
            f.optparser = SubCmdOptionParser()
        for k,v in kwargs.items():
            setattr(f.optparser, k, v)
        return f
    return decorate

class Mantash(cmdln.Cmdln):
    r"""${name} -- a Manta shell

    Usage:
        ${name} [<options>] <command> [<args>]
        ${name} help [<command>]
        ${name}                        # interactive shell

    ${option_list}
    Note on -k/MANTA_KEY_ID: node-manta (and possibly other manta tools) do
    not support the key id being a *path*. The advantage of a path is that a
    key path other than "~/.ssh/id_rsa" can be used without it being in your
    ssh-agent. Use a fingerprint (calculate as in the example below) if you
    are able, for compatibility.

    A reasonable starter environment might be:

        export MANTA_URL=https://us-east.manta.joyent.com
        export MANTA_USER=jill
        export MANTA_KEY_ID=`ssh-keygen -l -f ~/.ssh/id_rsa.pub | awk '{print $2}' | tr -d '\\n'`
        export MANTASH_PS1='\e[90m[\u@\h \e[34m\w\e[90m]$\e[0m '

    ${command_list}
    ${help_list}
    """
    name = "mantash"
    version = manta.__version__
    manta_url = None
    cwd = None          # initialized in `postoptparse`
    last_cwd = None     # ditto

    def get_optparser(self):
        parser = cmdln.Cmdln.get_optparser(self)
        #parser.add_option("--version", action="store_true",
        #    help="print mantash version and exit")
        parser.add_option("-v", "--verbose", dest="verbose",
            action="store_true", help="Verbose/debug logging.")
        parser.add_option("-u", "--url", dest="manta_url",
            help="Manta URL. Environment: MANTA_URL=URL",
            default=os.environ.get("MANTA_URL"))
        parser.add_option("-a", "--account", dest="account",
            help="Manta account (login name). Environment: MANTA_USER=ACCOUNT",
            default=os.environ.get("MANTA_USER"))
        parser.add_option("-k", "--keyId", dest="key_id",
            help="SSH key fingerprint (or path to private key file). See "
                "note below. Environment: MANTA_KEY_ID=FINGERPRINT",
            default=os.environ.get("MANTA_KEY_ID"))
        parser.add_option("-i", "--insecure", action="store_true",
            dest="insecure",
            help="Do not validate SSL/TLS certificate. Not recommended but "
                "useful for dev. Environment: MANTA_TLS_INSECURE=1",
            default=(os.environ.get("MANTA_TLS_INSECURE") == "1"))
        parser.add_option("-C", dest="cd", metavar="DIRECTORY",
            help="first change to the given directory")
        parser.add_option("--drop-cache", dest="drop_cache", action="store_true",
            help="drop the current HTTP cache before starting")
        return parser

    def postoptparse(self):
        if self.options.verbose:
            log.setLevel(logging.DEBUG)
        #log.debug("cache dir: %s", g_cache_dir)

        # HACK: Don't run the code below if just doing a 'help' command,
        # because then we might error out when the user is just trying to
        # get help.
        if len(sys.argv) > 1 and sys.argv[1] == 'help':
            return

        # TODO: would be preferable to have a '--no-cache' option that
        # just fully disabled using a cache. Perhaps setting the cache dir
        # to /dev/null? Or the MantaClient.no_cache thing above.
        if self.options.drop_cache and os.path.exists(HTTP_CACHE_DIR):
            import shutil
            shutil.rmtree(HTTP_CACHE_DIR)
            assert not os.path.exists(HTTP_CACHE_DIR)

        no_auth = (os.environ.get("MANTA_NO_AUTH", "false") == "true")
        if not self.options.manta_url:
            raise MantashError("no Manta URL: use '--url' or 'MANTA_URL' envvar")
        if not self.options.account:
            raise MantashError("no Manta account: use '--account' or 'MANTA_USER' envvar")
        if not no_auth and not self.options.key_id:
            raise MantashError("no Manta key ID: use '-i' or 'MANTA_KEY_ID' envvar")
        self.manta_url = self.options.manta_url

        self.ps1 = os.environ.get("MANTASH_PS1", DEFAULT_PS1)

        if (not self.manta_url.startswith('http://') and
            not self.manta_url.startswith('https://')):
            self.manta_url = 'https://' + self.manta_url
        if self.manta_url.endswith('/'):
            self.manta_url = self.manta_url[:-1]
        self.host = urlparse(self.manta_url).hostname

        self.account = self.options.account
        self.home = "/%s/stor" % self.account
        self.last_cwd = self.cwd = self.home
        if no_auth:
            signer = None
        else:
            signer = manta.CLISigner(self.options.key_id)
        self.client = manta.MantaClient(self.manta_url, self.account,
            signer=signer,
            disable_ssl_certificate_validation=self.options.insecure,
            user_agent=USER_AGENT, verbose=self.options.verbose)

        self.do_help.aliases.append("man")

        f = None
        self._known_users = {}
        try:
            f = codecs.open(KNOWN_USERS_PATH, 'r', 'utf8')
            self._known_users = json.load(f, encoding='utf8')
        except Exception, ex:
            pass
        finally:
            if f:
                f.close()

        self._update_prompt()

        if self.options.cd:
            return self.cmd(["cd", self.options.cd])

    def do_help(self, argv):
        if self.cmdlooping and len(argv) <= 1:
            doc = "${command_list}"
            doc = self._help_preprocess(doc, None)
            doc = doc.rstrip('\n')
            print(doc)
        else:
            cmdln.Cmdln.do_help(self, argv)
    do_help.aliases = cmdln.Cmdln.do_help.aliases

    def _update_prompt(self):
        """Supported MANTASH_PS1 codes:
            \h     the hostname up to the first `.'
            \H     the hostname
            \n     newline
            \r     carriage return
            \w     the current working directory
            \W     the basename of the current working directory
            \e     an ASCII escape character (033)
            \\     a backslash

        Currently not supported (because I'd need to hack cmdln.py a bit):
            \t     the current time in 24-hour HH:MM:SS format
        """
        if re.search('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', self.host):
            shorthost = self.host
        else:
            shorthost = self.host.split('.')[0]
        codes = {
            'm': self.manta_url,
            'w': self.cwd,
            'W': ubasename(self.cwd),
            'u': self.account,
            'h': shorthost,
            'H': self.host,
            'n': '\n',
            #'t': time.strftime('%H:%M:%S', time.localtime()),
            'v': '.'.join(self.version.split('.')[:2]),
            'V': self.version,
            'e': '\033',
            '\\': '\\',
        }
        def repl(match):
            return codes[match.group(1)]
        allkeys = ''.join(codes.keys()).replace('\\', '\\\\')
        uprompt = re.sub(r'\\([%s])' % allkeys, repl, self.ps1)
        self._prompt_str = uprompt.encode(
            sys.stdout.encoding or 'UTF-8', 'replace')

    def do_lpwd(self, subcmd, opts):
        """${cmd_name}: print the local cwd"""
        print(os.getcwd())

    def do_lcd(self, subcmd, opts, directory=None):
        """${cmd_name}: change the local cwd

        Usage:
            ${cmd_name} [DIR]
        """
        if not directory:
            lcwd = os.path.expanduser('~')
        else:
            lcwd = os.path.realpath(os.path.expanduser(directory))
        os.chdir(lcwd)

    def do_lls(self, argv):
        """${cmd_name}: local ls

        Usage:
            ${cmd_name} [ARGS]
        """
        os.system('ls %s' % ' '.join(argv[1:]))

    def do_pwd(self, subcmd, opts):
        """${cmd_name}: print the cwd"""
        print(self.cwd)

    def _get_dir(self, mdir):
        """Light wrapper around `self.client.ls(mdir)`.

        success -> dirents
        no such user -> raise MantashError
        404 -> None
        error -> raise MantaError
        """
        try:
            return self.client.ls(mdir)
        except manta.MantaAPIError:
            _, ex, _ = sys.exc_info()
            code = hasattr(ex, 'code') and ex.code
            if code == 'UserDoesNotExist':
                raise MantashError(str(ex))
            elif code == 'ResourceNotFound':
                return None
            else:
                raise

    _is_glob_re = re.compile(r'(?<!\\)[*?\[]')
    def _ls_path(self, path, listDirs):
        """Get the set of directory entries (dirents) matching the
        given path, which may be a glob pattern.

        @param path {str} Path or path glob pattern.
        @param listDirs {boolean} List directory names, do not search recursively.
        @raises {MantashError} For any error ls'ing.
        @returns list of (<is-dir>, <path>, <dirents>), e.g.:
            [(False,
              'tmp/*.txt',
              {'tmp/foo.txt': {'name': 'foo.txt', 'type': 'object'}}),
             (True,
              'lib',
              {'bar.py': {'mtime': 1341422663102L, 'type': 'object'},
               'baz': {'mtime': 1341422663102L, 'type': 'directory'}})]
        """
        upath = uexpanduser(path, self.home)    # user-expanded
        npath = unormpath(upath)                # normalized
        abspath = unormpath(ujoin(self.cwd, npath))
        absparts = abspath.split('/')
        parts = upath.split('/')
        is_glob = self._is_glob_re.search(path) is not None
        head = '/'.join(absparts[:3])
        if self._is_glob_re.search(head):
            raise MantashError("cannot use glob chars in first two parts "
                "of a Manta path: %r" % head)

        DEBUG = False
        if DEBUG:
            print("-- _ls_path")
            print("  path: %r" % path)
            print("  upath: %r" % upath)
            print("  parts: %r" % parts)
            print("  npath: %r" % npath)
            print("  abspath: %r" % abspath)
            print("  absparts: %r" % absparts)
            print("  is_glob: %r" % is_glob)
            print("  head: %r" % head)

        if abspath == '/':
            is_dir = True
            if listDirs:
                dirents = {abspath: {"name": "/", "type": "directory"}}
            else:
                # TODO: Just the user for now. Add cached visited public
                # users eventually.
                dirents = {
                    self.account: {"name": self.account, "type": "directory"}
                }
            hits = [(is_dir, upath, dirents)]

        elif len(absparts) == 2:   # '/USER'
            # '/USER'. Just presume that user exists for now.
            is_dir = True
            if listDirs:
                dirents = {abspath: {"name": absparts[1], "type": "directory"}}
            elif absparts[1] == self.account:
                dirents = {
                    "stor": {"name": "stor", "type": "directory"},
                    "public": {"name": "public", "type": "directory"},
                    "jobs": {"name": "jobs", "type": "directory"},
                    "reports": {"name": "reports", "type": "directory"},
                }
            else:
                dirents = {
                    "public": {"name": "public", "type": "directory"}
                }
            hits = [(is_dir, upath, dirents)]

        elif not is_glob:
            # Get the dirent for this path.
            if len(absparts) == 3:
                # '/USER/{stor,public,jobs,reports}'
                if absparts[2] not in ("stor", "public", "jobs", "reports"):
                    raise MantashError("%s: invalid manta path" %
                        (path or '/'))
                dirent = {"name": absparts[2], "type": "directory"}
            else:
                # Ensure it exists.
                parent = udirname(abspath)
                name = ubasename(abspath)
                pdirents = self._get_dir(parent)
                if pdirents is None:
                    dirent = None
                else:
                    dirent = pdirents.get(name)
            if not dirent:
                raise MantashError("%s: no such object or directory" %
                    (path or '/'))
            if DEBUG:
                print("  dirent: %r" % dirent)

            if dirent.get("type") == "directory":
                is_dir = True
                if listDirs:
                    dirents = {
                        upath: dirent
                    }
                else:
                    dirents = self.client.ls(abspath)
            else:
                is_dir = False
                dirents = {
                    upath: dirent
                }
            hits = [(is_dir, upath, dirents)]

        else:
            # Work up to first dir with a glob char.
            lead = absparts[:3]  # no glob in first two segments allowed
            for i in range(3, len(absparts)-1):
                if self._is_glob_re.search(absparts[i]):
                    break
                lead.append(absparts[i])
            absprefix = '/'.join(lead) + '/'
            prefix = []
            for i in range(len(parts)):
                if self._is_glob_re.search(parts[i]):
                    break
                prefix.append(parts[i])
            prefix = '/'.join(prefix)
            if prefix:
                prefix += '/'
            if DEBUG:
                print("  non-glob dirs lead: %r" % lead)
                print("  absprefix: %r" % absprefix)
                print("  prefix: %r" % prefix)

            # `matches` is a mapping of {match -> dirent} where each
            # match is a base dir for next iteration.
            matches = {'/'.join(lead): {"type": "directory"}}
            for i in range(len(lead), len(absparts)):
                p = absparts[i]
                patterns = [unormpath(ujoin(m, p)) for m,dirent in matches.items()
                    if dirent["type"] == "directory"]
                if DEBUG:
                    print("  glob part %r: patterns %s from matches %r" % (p, patterns, matches))
                matches = {}
                for pattern in patterns:
                    parent = udirname(pattern)
                    basepat = ubasename(pattern)
                    dirents = self._get_dir(ujoin(self.cwd, parent))
                    if DEBUG:
                        print("    pattern %r:\n\tparent %r\n\tbasepat %r\n\tdirents %r" % (pattern, parent, basepat, dirents))
                    if dirents:
                        for name, dirent in dirents.items():
                            if fnmatch.fnmatchcase(name, basepat):
                                matches[ujoin(parent, name)] = dirent
                            elif DEBUG:
                                print("    discard %r (does not match %r)" % (name, basepat))
                if DEBUG:
                    print("    new matches: %r" % matches)
                if not matches:
                    raise MantashError("%s: no such object or directory" %
                        (path or '/'))

            # Any leaf directories are recursed into... unless `ls -d`.
            #
            # Note: We did globbing on the abspath, make the paths relative
            # again (if the given start path was relative).
            hits = []
            if not listDirs:
                for name,dirent in matches.items():
                    if dirent["type"] == "directory":
                        if DEBUG:
                            print("    descend into dir %r" % name)
                        del matches[name]
                        dirents = self._get_dir(name)
                        relname = name.replace(absprefix, prefix, 1)
                        hits.append((True, relname, dirents))
            if matches:
                relmatches = {}
                for key in matches.keys():
                    if key.startswith(absprefix):
                        relkey = key.replace(absprefix, prefix, 1)
                        relmatches[relkey] = matches[key]
                hits.append((False, upath, relmatches))

        if hits and hits[0][-1]:
            self._visited_known_mpath(abspath)
        if DEBUG:
            print("  hits:\n%s" % _indent(pformat(hits)))
        return hits

    @cmdln.option("-h", action="store_true", dest="human",
        help="display human-readable sizes")
    @optparser_setattr(conflict_handler="resolve")
    @cmdln.option("-s", action="store_true", dest="summary",
        help="display an entry for each specified file")
    def do_du(self, subcmd, opts, *paths):
        """disk usage

        Usage:
            ${cmd_name} [OPTIONS...] [PATHS...]

        ${cmd_option_list}
        """
        retval = None  # set to 1 for any errors

        def du(dirent):
            if opts.human:
                dirent["size"] = self._human_size(dirent["size"])
                print("%(size)-10s %(path)s" % dirent)
            else:
                print("%(size)-10d %(path)s" % dirent)

        # First pass local globbing. Really this should be a `client.ftw`.
        dirents = []
        for path in paths:
            npath = unormpath(ujoin(self.cwd, uexpanduser(path, self.home)))
            d, b = usplit(npath)
            if '*' in b or '?' in b or '[' in b: # we have a glob
                candidates = self.client.ls(d)
                for match in fnmatch.filter(candidates.keys(), b):
                    e = candidates[match]
                    e["path"] = ujoin(d, match)
                    dirents.append(e)
            else:
                try:
                    stat = self.client.stat(npath)
                except manta.MantaResourceNotFoundError:
                    log.warn("'%s' not found", path)
                    continue
                stat = self.client.stat(npath)
                stat["path"] = npath
                dirents.append(stat)

        for dirent in dirents:
            path = dirent["path"]
            if dirent["type"] == "directory":
                if opts.summary:
                    total = 0
                    for d, dirents, objents in self.client.walk(path, False):
                        for objent in objents:
                            total += objent["size"]
                    dirent["size"] = total
                    du(dirent)
                else:
                    for d, dirents, objents in self.client.walk(path, False):
                        for objent in objents:
                            objent["path"] = ujoin(d, objent["name"])
                            du(objent)
            else:
                du(dirent)


    @cmdln.option("-h", action="store_true", dest="human",
        help="display human-readable sizes")
    @cmdln.option("-a", action="store_true", help=optparse.SUPPRESS_HELP)
    @cmdln.option("-l", action="store_true", dest="long",
        help="list in long format")
    @cmdln.option("-d", action="store_true", dest="dir",
        help="list directory names, do not search recursively")
    @cmdln.option("-F", action="store_true", dest="format",
        help="display a '/' after a directory name")
    @optparser_setattr(conflict_handler="resolve")
    @cmdln.option("-j", "--json", action="store_true",
        help="display the listing in JSON")
    def do_ls(self, subcmd, opts, *paths):
        """list objects, directories and links

        Usage:
            ${cmd_name} [OPTIONS...] [PATHS...]

        ${cmd_option_list}
        """
        retval = None  # set to 1 for any errors

        # Gather list of things to print.
        paths = paths or ['']
        all_to_print = []  # list of (<is-dir>, <path>, <dirents>)
        for path in paths:
            upath = path.decode(sys.stdin.encoding)
            try:
                to_print = self._ls_path(path, opts.dir)
            except (MantaError, MantashError):
                _, ex, _ = sys.exc_info()
                log.error(ex)
                retval = 1
            else:
                all_to_print += to_print

        # Do non-dirs first, as does `ls dirA fileB dirC fileD`.
        if opts.json:
            listing = {}
        for i, (is_dir, path, dirents) in enumerate(
                sorted(all_to_print, key=itemgetter(0))):
            if opts.json:
                listing[path] = dirents
                continue
            if is_dir and len(all_to_print) > 1:
                if i > 0:
                    print("")
                print("%s:" % path)
            for name, dirent in sorted(dirents.iteritems()):
                self._ls_print_dirent(name, dirent, long=opts.long,
                    format=opts.format, human=opts.human)
        if opts.json:
            print(json.dumps(listing, indent=2))
        return retval

    def _ls_print_dirent(self, name, dirent, long=False, format=False,
                         human=False):
        if format:
            name += {
                "directory": "/",
                "link": "@"
            }.get(dirent["type"], "")
        if long:
            if human:
                dirent_fmt = "%9s  %s  %04s  %24s  %s"
            else:
                dirent_fmt = "%9s  %s  %10s  %24s  %s"

            sz = dirent.get("size", 0)
            if (sz != "-" and human):
                sz = self._human_size(sz)

            # Note: Don't have a strong reason to show the "asctime"
            # format here, rather than just parroting the RFC 3339
            # format. IOW, if others have opinions on the time format here,
            # I'm fine changing.
            mtime = dirent.get('mtime', '')
            #if "mtime" in dirent:
            #    # AFAIK Python's strptime doesn't support milliseconds.
            #    mtime = time.asctime(
            #        time.strptime(dirent["mtime"], "%Y-%m-%dT%H:%M:%S.721Z"))
            print(dirent_fmt % (
                dirent.get("type", "unknown"), self.account, sz, mtime, name))
        else:
            print(name)

    def _human_size(self, nbytes):
        suffixes = ['', 'K', 'M', 'G', 'T', 'P', 'E']

        size = float(nbytes)
        for suffix in suffixes:
            if size < 1024:
                break
            size /= 1024.0

        # If size < 10, show a single decimal value.  Otherwise don't show any.
        if size < 10 and suffix:
            size = '%.1f' % size
        else:
            size = '%d' % size

        return size + suffix

    def _visited_known_mpath(self, mpath):
        """Subcommands should call this whenever having learned that a given
        Manta path exists. It is used to assist with tab-completion of Manta
        paths -- currently just for completion of user dirs (the first
        path component).
        """
        parts = mpath.split('/')
        if len(parts) < 3:
            # No dir to save if only at top-level.
            return
        user = parts[1]
        area = parts[2]
        if area != 'public':
            area = 'private'
        if (area == 'public' and user not in self._known_users
            or self._known_users.get('user') != 'private'):
            self._known_users[user] = area
            f = None
            try:
                f = codecs.open(KNOWN_USERS_PATH, 'w', 'utf8')
                json.dump(self._known_users, f)
            except Exception:
                pass
            finally:
                if f:
                    f.close()

    def do_cd(self, subcmd, opts, directory='~'):
        """change directory

        Usage:
            ${cmd_name} [DIRECTORY]
        """
        udirectory = directory.decode(sys.stdin.encoding)
        ndir = unormpath(ujoin(self.cwd, uexpanduser(udirectory, self.home)))
        nparts = ndir.split('/')
        last_cwd = self.cwd
        if directory == '-':
            print(self.last_cwd)
            self.cwd = self.last_cwd
        elif len(nparts) <= 2:
            # '/' or '/USER'
            self.cwd = ndir
        elif len(nparts) == 3:
            # '/USER/SCOPE'
            if nparts[2] not in ("stor", "public", "reports", "jobs"):
                log.error("%s: no such directory", directory)
                return 1
            self.cwd = ndir
        else:
            parent = udirname(ndir)
            name = ubasename(ndir)
            try:
                dirents = self.client.ls(parent)
            except manta.MantaAPIError, ex:
                if ex.code == 'ResourceNotFound':
                    log.error("%s: no such directory", directory)
                    return 1
                raise
            if name not in dirents:
                log.error("%s: no such directory", directory)
                return 1
            elif dirents[name].get("type") != "directory":
                log.error("%s: not a directory", directory)
                return 1
            else:
                self._visited_known_mpath(ndir)
                self.cwd = ndir
        self.last_cwd = last_cwd
        self._update_prompt()

    def completedefault(self, text, line, begidx, endidx):
        """Complete paths in the cwd."""
        try:
            DEBUG = False
            if DEBUG:
                print("\n-- completedefault:",)
                print("  %r" % ((text, line, begidx, endidx),))
            subcmd = line and line.split(None, 1)[0]
            start = line[begidx:endidx]
            if DEBUG:
                print("  subcmd: %r" % subcmd)
                print("  start: %r" % start)
            mode = {
                "cat": "manta-path",
                "cd": "manta-dir",
                "find": "manta-dir",
                # The *last* arg for 'get' is a local path, but there can
                # be N leading manta-paths, so no perfect completion answer
                # here. Could complete both styles for second+ arg, but that
                # is confusing.
                "get": "manta-path",
                "job": "manta-path",
                "lcd": "local-dir",
                "lls": "local-path",
                "ln": "manta-path",
                "ls": "manta-path",
                "mkdir": "manta-dir",
                # Similar to 'get', no good way to know when changing from
                # local path(s) to manta path.
                "put": "local-path",
                "rm": "manta-path",
                "vi": "manta-path",
                "jobinfo": "job-id",
                "login": "manta-path"
            }.get(subcmd, "manta-path")

            startbase = None
            if mode == "job-id":
                matches = [j["id"] for j in self.client.list_jobs(limit=100)]
            elif start in ("~", "~/"):
                if mode.startswith("manta"):
                    matches = [self.home + '/']
                else:
                    matches = [os.path.expanduser(start) + os.sep]
            elif mode in ("manta-path", "manta-dir"):
                if start.startswith("~"):
                    start = uexpanduser(start, self.home)
                startdir = udirname(start)
                startbase = ubasename(start)
                parent = unormpath(ujoin(self.cwd, udirname(start)))
                if parent == '/':
                    matches = ['/' + u + '/' for u in self._known_users.keys()
                        if u.startswith(startbase)]
                elif parent.count('/') == 1:
                    # Infer if have private access from previous successful
                    # operations on files in private dirs.
                    account = self.cwd.split('/')[1]
                    priv_access = (account == self.account
                        or self._known_users.get(account) == 'private')
                    if priv_access:
                        dirs = ['jobs', 'public', 'reports', 'stor']
                    else:
                        dirs = ['public']
                    matches = [ujoin(startdir, n) + '/' for n in dirs
                        if n.startswith(startbase)]
                else:
                    dirents = self._get_dir(parent)
                    if DEBUG:
                        print("  dirents: %r" % dirents)
                    if not dirents:
                        matches = []
                    elif mode == "manta-dir":
                        matches = [
                            ujoin(startdir, n) + '/' for n,d in dirents.items()
                            if n.startswith(startbase)
                            and n.startswith('.') == startbase.startswith('.')
                            and d["type"] == "directory"]
                    else:
                        matches = [
                            ujoin(startdir, n) + (d["type"] == "directory" and "/" or "")
                            for n,d in dirents.items()
                            if n.startswith(startbase)
                            and n.startswith('.') == startbase.startswith('.')]
            elif mode in ("local-path", "local-dir"):
                start = os.path.expanduser(start)
                startdir = os.path.dirname(start)
                startbase = os.path.basename(start)
                parent = os.path.join(os.getcwd(), startdir)
                matches = [os.path.join(startdir, n) for n in os.listdir(parent)
                           if n.startswith(startbase)
                           and not n.startswith('.')]
                if mode == "local-dir":
                    matches = [m + '/' for m in matches if os.path.isdir(m)]
                else:
                    matches = [m + (os.path.isdir(m) and '/' or '')
                        for m in matches]
            if startbase == '.':
                matches.append('.')
                matches.append('..')
            elif startbase == '..':
                matches.append('..')

            # Escape matches for the shell.
            # The list of special chars from:
            # https://pangea.stanford.edu/computing/unix/shell/specialchars.php
            # - I excluded '~' from that list because don't want to escape '~'
            #   at the start of a path and not sure it is necessary elsewhere.
            # - Also excluded '-', b/c it is not necessary.
            escaper = re.compile(r'''([][ \\'"*?{}$!&;()<>|#@:])''')
            for i, match in enumerate(matches):
                matches[i] = escaper.sub(r'\\\1', match)

            # UTF-8 encode results for GNU readline.
            for i, match in enumerate(matches):
                matches[i] = match.encode('utf-8')

            matches.sort()
            if DEBUG:
                print("  %d matches: %r" % (len(matches), matches))
                print("\t%s" % '\n\t'.join(matches))
            return matches
        except:
            if DEBUG or self.options.debug:
                import traceback
                traceback.print_exc()
            raise

    def do_cat(self, subcmd, opts, *paths):
        """Download Manta objects and print them to stdout.

        Usage:
            ${cmd_name} [MANTA-PATHS...]

        ${cmd_option_list}
        """
        retval = None
        content = None
        for path in paths:
            #TODO: check its dir to make sure it is an object
            npath = unormpath(ujoin(self.cwd, uexpanduser(path, self.home)))
            parent = udirname(npath)
            name = ubasename(npath)
            dirents = self._get_dir(parent)
            if name not in dirents:
                log.error("%s: no such object or directory", path)
                retval = 1
                continue
            elif dirents[name]["type"] == "directory":
                log.error("%s: is a directory", path)
                retval = 1
                continue
            content = self.client.get(npath)
            sys.stdout.write(content)
        if content and not content.endswith('\n'):
            sys.stdout.write('\n')
        return retval

    def do_head(self, subcmd, opts, *paths):
        """Download Manta objects and print the first N lines to stdout.

        Usage:
            ${cmd_name} [MANTA-PATHS...]

        ${cmd_option_list}
        """
        retval = None
        head = None
        n = 10   # TODO: option for '-n'. optional '-10' support?
        for i, path in enumerate(paths):
            #TODO: check its dir to make sure it is an object
            npath = unormpath(ujoin(self.cwd, uexpanduser(path, self.home)))
            parent = udirname(npath)
            name = ubasename(npath)
            dirents = self._get_dir(parent)
            if name not in dirents:
                log.error("%s: no such object or directory", path)
                retval = 1
                continue
            elif dirents[name]["type"] == "directory":
                log.error("%s: is a directory", path)
                retval = 1
                continue
            content = self.client.get(npath)
            if len(paths) > 1:
                if i != 0:
                    print
                print '==> %s <==' % npath
            head = ''.join(content.splitlines(True)[:n])
            sys.stdout.write(head)
            if head and not head.endswith('\n'):
                sys.stdout.write('\n')
        return retval

    def do_tail(self, subcmd, opts, *paths):
        """Download Manta objects and print the last N lines to stdout.

        Usage:
            ${cmd_name} [MANTA-PATHS...]

        ${cmd_option_list}
        """
        retval = None
        tail = None
        n = 10   # TODO: option for '-n'. optional '-10' support?
        for i, path in enumerate(paths):
            #TODO: check its dir to make sure it is an object
            npath = unormpath(ujoin(self.cwd, uexpanduser(path, self.home)))
            parent = udirname(npath)
            name = ubasename(npath)
            dirents = self._get_dir(parent)
            if name not in dirents:
                log.error("%s: no such object or directory", path)
                retval = 1
                continue
            elif dirents[name]["type"] == "directory":
                log.error("%s: is a directory", path)
                retval = 1
                continue
            content = self.client.get(npath)
            if len(paths) > 1:
                if i != 0:
                    print
                print '==> %s <==' % npath
            tail = ''.join(content.splitlines(True)[-n:])
            sys.stdout.write(tail)
            if tail and not tail.endswith('\n'):
                sys.stdout.write('\n')
        return retval


    def do_json(self, subcmd, opts, path):
        """Download a Manta objects and JSON pretty-print it.
        Note: This is a precursor to nicely passing this to a full `json`.

        Usage:
            ${cmd_name} MANTA-PATH

        ${cmd_option_list}
        """
        retval = None
        npath = unormpath(ujoin(self.cwd, uexpanduser(path, self.home)))
        parent = udirname(npath)
        name = ubasename(npath)
        dirents = self._get_dir(parent)
        if name not in dirents:
            log.error("%s: no such object or directory", path)
            return 1
        elif dirents[name]["type"] == "directory":
            log.error("%s: is a directory", path)
            return 1
        content = self.client.get(npath)
        try:
            parsed = json.loads(content)
        except Exception:
            sys.stdout.write(content)
            if content and not content.endswith('\n'):
                sys.stdout.write('\n')
            return 1
        else:
            sys.stdout.write(json.dumps(parsed, indent=2) + '\n')

    def do_zcat(self, subcmd, opts, *paths):
        """Download Manta objects, gunzip and print them to stdout.

        Usage:
            ${cmd_name} [MANTA-PATHS...]

        ${cmd_option_list}
        """
        from cStringIO import StringIO
        import gzip

        retval = None
        content = None
        for path in paths:
            #TODO: check its dir to make sure it is an object
            npath = unormpath(ujoin(self.cwd, uexpanduser(path, self.home)))
            parent = udirname(npath)
            name = ubasename(npath)
            dirents = self._get_dir(parent)
            if name not in dirents:
                log.error("%s: no such object or directory", path)
                retval = 1
                continue
            elif dirents[name]["type"] == "directory":
                log.error("%s: is a directory", path)
                retval = 1
                continue
            content = self.client.get(npath)
            f = StringIO(content)
            unzipper = gzip.GzipFile(npath, 'rb', fileobj=f)
            sys.stdout.write(unzipper.read())
            f.close()
        if content and not content.endswith('\n'):
            sys.stdout.write('\n')
        return retval

    def do_open(self, subcmd, opts, path):
        """open the given manta path in your browser

        Usage:
            ${cmd_name} [OPTIONS...] MANTA-PATH

        ${cmd_option_list}
        """
        upath = path.decode(sys.stdin.encoding)
        npath = unormpath(ujoin(self.cwd, uexpanduser(upath, self.home)))
        url = self.manta_url + npath
        webbrowser.open(url.encode('utf-8'))

    def do_exit(self, subcmd, opts):
        """${cmd_name}: exit the shell"""
        print("exit")
        self.stop = True

    @cmdln.option("-p", dest="parents", action="store_true",
        help="create intermediate directories as require")
    def do_mkdir(self, subcmd, opts, *paths):
        """create a directory

        Usage:
            ${cmd_name} PATH ...

        ${cmd_option_list}
        """
        if not paths:
            log.error("mkdir: no PATH arguments given")
            return 1
        retval = None
        for path in paths:
            mdir = unormpath(ujoin(self.cwd, uexpanduser(path, self.home)))
            try:
                self.client.mkdir(mdir, parents=opts.parents)
            except manta.MantaError:
                _, ex, _ = sys.exc_info()
                log.error(ex)
                retval = 1
        return retval

    @cmdln.option("-v", "--verbose", action="store_true",
        help="show files (and content-type) as they are being copied")
    @cmdln.option("-d", "--durability-level",
        help="Tell Manta the number of copies to keep. Default is 2.")
    @cmdln.option("-t", "--content-type",
        help="specify content-type, else this will attempt to guess based "
            "on the filename. This 'wins' over '-b'.")
    @cmdln.option("-b", "--binary", action="store_true",
        help="disable content-type guessing and use 'application/octet-stream'")
    @cmdln.option("-R", "-r", dest="recursive", action="store_true",
        help="recursively copy a source directory")
    @cmdln.option("--dry-run", action="store_true",
        help="do a dry-run, implies '--verbose'")
    def do_put(self, subcmd, opts, *paths):
        """put a local file to manta

        Usage:
            ${cmd_name} [OPTIONS] LOCAL-PATH MANTA-PATH
            ${cmd_name} [OPTIONS] LOCAL-PATH ... MANTA-DIRECTORY

        ${cmd_option_list}
        """
        if len(paths) < 2:
            log.error("incorrect number of arguments")
            return 1
        if opts.dry_run:
            opts.verbose = True
        src_paths = paths[:-1]
        dst_path = paths[-1]

        # In all cases we get the normalized dest path.
        # Non-recursive:
        #   cp file1 file2          find out if dest is a dir
        #   cp file1 dir2           ditto, dest=dir2/basename(file1)
        #   cp file1 dir2/          we know dest is a dir, assert that, dest=dir2/basename(file1)
        #   cp file1 file2 dir2     dest must be a dir, assert that, dest=...
        #   cp file1 file2 dir2/    dest must be a dir, assert that, dest=...
        # Recursive:
        #   cp -r file1 file2       find out if dest is a dir
        #   cp -r file1 dir2        ditto, dest=dir2/basename(file1)
        #   cp -r file1 dir2/       assert dir2 is a dir, dest=...
        #   cp -r dir1 dir2         if dir2 exists then dest=dir2/basename(dir1)/...
        #                               else dest=dir2/...
        #                               (*) only case where 'dir2' is created
        #   cp -r dir1/ dir2        dest=dir2/...
        #                               (*) only case where 'dir2' is created
        #   cp -r file1 file2 dir2  assert dir2 is a dir
        #   cp -r file1 dir1 dir2   assert dir2 is a dir

        # Get info on dest path
        # TODO: could switch to self._manta_isdir()
        dst_realpath = unormpath(ujoin(self.cwd,
            uexpanduser(dst_path, self.home)))
        dst_parts = dst_realpath.split('/')
        if len(dst_parts) <= 3:
            dst_is_existing_dir = True
        else:
            dst_parent = udirname(dst_realpath)
            dst_base = ubasename(dst_realpath)
            dirents = self._get_dir(dst_parent)
            if dirents is None:
                log.error("%s: no such remote directory", dst_parent)
                return 1
            dst_is_existing_dir = (dst_base in dirents
                and dirents[dst_base]["type"] == "directory")

        # Sanity checks on args.
        if len(src_paths) > 1 or dst_path.endswith('/'):
            # dst must be an existing dir
            if not dst_is_existing_dir:
                log.error("%s (remote) is not an existing directory", dst_path)
                return 1

        def put_file(src_file, dst_file):
            if opts.content_type:
                content_type = opts.content_type
            elif opts.binary:
                content_type = "application/octet-stream"
            else:
                content_type = (mimetypes.guess_type(src_file)[0]
                    or "application/octet-stream")

            if opts.verbose:
                log.info("put %s %s  # %s", src_file, dst_file, content_type)
            if not opts.dry_run:
                self.client.put(dst_file, path=src_file,
                    content_type=content_type,
                    durability_level=opts.durability_level)

        # Copy the files.
        retval = None
        for src_path in src_paths:
            if not opts.recursive:
                # Must be regular file.
                if not os.path.isfile(src_path):
                    log.error("%s is not a regular file (not copied)", src_path)
                    retval = 1
                    continue
                if dst_is_existing_dir:
                    dst_file = ujoin(dst_realpath, os.path.basename(src_path))
                    put_file(src_path, dst_file)
                else:
                    put_file(src_path, dst_realpath)
            elif os.path.isfile(src_path):
                if dst_is_existing_dir:
                    dst_file = ujoin(dst_realpath, os.path.basename(src_path))
                    put_file(src_path, dst_file)
                else:
                    put_file(src_path, dst_realpath)
            elif os.path.isdir(src_path):
                if not dst_is_existing_dir:
                    # `mkdir dst_realpath`. See '(*)' case above.
                    if not opts.dry_run:
                        self.client.mkdir(dst_realpath)

                rel_prefix = ((dst_is_existing_dir and not src_path.endswith('/'))
                    and os.path.basename(src_path) + '/' or '')
                if rel_prefix:
                    if not opts.dry_run:
                        self.client.mkdir(ujoin(dst_realpath, rel_prefix))
                norm_src_path = src_path.rstrip('/')
                for dirpath, dirnames, filenames in os.walk(norm_src_path):
                    reldirpath = unormpath(
                        rel_prefix + dirpath[len(norm_src_path)+1:])
                    for filename in filenames:
                        put_file(os.path.join(dirpath, filename), unormpath(
                            ujoin(dst_realpath, reldirpath, filename)))
                    if not opts.dry_run:
                        for dirname in dirnames:
                            self.client.mkdir(unormpath(
                                ujoin(dst_realpath, reldirpath, dirname)))
            else:
                log.error("%s is not a regular file or directory (not copied)",
                    src_path)
                retval = 1
                continue
        return retval

    @cmdln.option("-v", "--verbose", action="store_true",
        help="show files (and content-type) as they are being copied")
    @cmdln.option("-R", "-r", dest="recursive", action="store_true",
        help="recursively copy a source directory")
    @cmdln.option("--dry-run", action="store_true",
        help="do a dry-run, implies '--verbose'")
    def do_get(self, subcmd, opts, *paths):
        """get a file from manta

        Usage:
            ${cmd_name} [OPTIONS] MANTA-PATH [LOCAL-PATH]
            ${cmd_name} [OPTIONS] MANTA-PATHS... LOCAL-DIRECTORY

        ${cmd_option_list}
        """
        if len(paths) < 1:
            log.error("incorrect number of arguments")
            return 1
        elif len(paths) == 1:
            src_paths = paths
            dst_path = '.'
        else:
            src_paths = paths[:-1]
            dst_path = paths[-1]
        if opts.dry_run:
            opts.verbose = True

        # In all cases we get the normalized dest path.
        # Non-recursive:
        #   put file1 file2          find out if dest is a dir
        #   put file1 dir2           ditto, dest=dir2/basename(file1)
        #   put file1 dir2/          we know dest is a dir, assert that, dest=dir2/basename(file1)
        #   put file1 file2 dir2     dest must be a dir, assert that, dest=...
        #   put file1 file2 dir2/    dest must be a dir, assert that, dest=...
        # Recursive:
        #   put -r file1 file2       find out if dest is a dir
        #   put -r file1 dir2        ditto, dest=dir2/basename(file1)
        #   put -r file1 dir2/       assert dir2 is a dir, dest=...
        #   put -r dir1 dir2         if dir2 exists then dest=dir2/basename(dir1)/...
        #                               else dest=dir2/...
        #                               (*) only case where 'dir2' is created
        #   put -r dir1/ dir2        dest=dir2/...
        #                               (*) only case where 'dir2' is created
        #   put -r file1 file2 dir2  assert dir2 is a dir
        #   put -r file1 dir1 dir2   assert dir2 is a dir

        # Get info on dest path
        dst_normpath = os.path.normpath(os.path.join(os.getcwd(),
            os.path.expanduser(dst_path)))
        dst_is_existing_dir = os.path.isdir(dst_normpath)

        # Sanity checks on args.
        if len(src_paths) > 1 or dst_path.endswith('/'):
            # dst must be an existing dir
            if not dst_is_existing_dir:
                log.error("%s (local) is not an existing directory", dst_path)
                return 1

        def get_file(src_file, dst_file):
            if opts.verbose:
                log.info("get %s %s", src_file, dst_file)
            if not opts.dry_run:
                self.client.get(src_file, dst_file)

        # Copy the files.
        retval = None
        for src_path in src_paths:
            src_npath = unormpath(ujoin(self.cwd,
                uexpanduser(src_path, self.home)))
            src_type = self.client.type(src_npath)
            if not opts.recursive:
                # Must be manta object (as opposed to a dir).
                if src_type != "object":
                    log.error("%s is not a Manta object (not copied)", src_path)
                    retval = 1
                    continue
                if dst_is_existing_dir:
                    dst_file = os.path.join(dst_path, ubasename(src_npath))
                    get_file(src_npath, dst_file)
                else:
                    get_file(src_npath, dst_path)
            elif src_type == "object":
                if dst_is_existing_dir:
                    dst_file = os.path.join(dst_path, basename(src_npath))
                    get_file(src_npath, dst_file)
                else:
                    get_file(src_npath, dst_path)
            elif src_type == "directory":
                if not dst_is_existing_dir:
                    # `mkdir dst_path`. See '(*)' case above.
                    if not opts.dry_run:
                        log.info("mkdir %s", dst_path)
                        os.mkdir(dst_path)

                rel_prefix = ((dst_is_existing_dir and not src_path.endswith('/'))
                    and ubasename(src_npath) + '/' or '')
                if rel_prefix:
                    d = os.path.join(dst_path, rel_prefix)
                    if not opts.dry_run:
                        log.info("mkdir %s", d)
                        os.mkdir(d)
                for dirpath, dirents, objents in self.client.walk(src_npath):
                    #pprint((dirpath, dirents, objents))
                    reldirpath = os.path.normpath(
                        rel_prefix + dirpath[len(src_npath)+1:])
                    for objent in objents:
                        get_file(ujoin(dirpath, objent["name"]),
                            os.path.normpath(os.path.join(
                                dst_path, reldirpath, objent["name"])))
                    for dirent in dirents:
                        if not opts.dry_run:
                            d = os.path.normpath(os.path.join(
                                dst_path, reldirpath, dirent["name"]))
                            log.debug("mkdir %s", d)
                            os.mkdir(d)
            elif src_type is None:
                log.error("%s (manta path) does not exist", src_path)
                retval = 1
                continue
            else:
                log.error("%s (manta path) is of unknown type, not an object "
                    "or directory (not copied)", src_path)
                retval = 1
                continue
        return retval

    @cmdln.option("-v", "--verbose", action="store_true",
        help="show files as they are being removed")
    @cmdln.option("-f", "--force", action="store_true",
        help="force removal, don't complain about file not existing")
    @cmdln.option("-r", "--recursive", action="store_true",
        help="recursively delete a directory")
    @cmdln.option("--dry-run", action="store_true",
        help="do a dry-run, implies '--verbose'")
    def do_rm(self, subcmd, opts, *paths):
        """rm a file from manta

        Usage:
            ${cmd_name} MANTA-PATHS...

        ${cmd_option_list}
        """
        if opts.dry_run:
            opts.verbose = True
        retval = None

        def remove_thing(mpath):
            if opts.verbose:
                log.info("rm %s", mpath)
            if not opts.dry_run:
                try:
                    self.client.rm(mpath)
                except manta.MantaAPIError:
                    _, ex, _ = sys.exc_info()
                    if opts.force and ex.code == 'ResourceNotFound':
                        pass
                    else:
                        log.error("rm %s: %s", mpath, ex)
                        retval = 1

        # First pass local globbing. Really this should be a `client.ftw`.
        dirents = []
        for path in paths:
            npath = unormpath(ujoin(self.cwd, uexpanduser(path, self.home)))
            d, b = usplit(npath)
            if '*' in b or '?' in b or '[' in b: # we have a glob
                candidates = self.client.ls(d)
                for match in fnmatch.filter(candidates.keys(), b):
                    e = candidates[match]
                    e["path"] = ujoin(d, match)
                    dirents.append(e)
            else:
                try:
                    stat = self.client.stat(npath)
                except manta.MantaResourceNotFoundError:
                    log.warn("'%s' not found", path)
                    continue
                stat = self.client.stat(npath)
                stat["path"] = npath
                dirents.append(stat)

        for dirent in dirents:
            path = dirent["path"]
            if not opts.recursive or not dirent["type"] == "directory":
                remove_thing(path)
            else:
                # Recursive delete of a directory.
                for dirpath, dirents, objents in self.client.walk(path, False):
                    #pprint((dirpath, dirents, objents))
                    for objent in objents:
                        remove_thing(ujoin(dirpath, objent["name"]))
                    remove_thing(dirpath)

        return retval

    @cmdln.option("-v", "--verbose",
        dest="verbose", action="store_true", default=True,
        help="show files as they are being moved. This is the default.")
    @cmdln.option("-q", "--quiet",
        dest="verbose", action="store_false",
        help="show files as they are being moved")
    @cmdln.option("-f", "--force", action="store_true",
        help="do not stop on overwriting a target file")
    @cmdln.option("--dry-run", action="store_true", help="do a dry-run")
    def do_mv(self, subcmd, opts, *paths):
        """move file(s)/dir(s) in manta

        Usage:
            ${cmd_name} MANTA-SOURCE MANTA-DEST
            ${cmd_name} MANTA-SOURCE ... MANTA-DEST-DIRECTORY

        ${cmd_option_list}

        Note: The eventual goal is to behave like a typical `mv` but for now
        this will error out if it hits target files in the way.
        """
        def move_dir(a, b):
            if opts.verbose:
                log.info("mkdir -p %s", b)
            if not opts.dry_run:
                self.client.mkdirp(b)

        def move_obj(a, b):
            if opts.verbose:
                log.info("mv %s %s", a, b)
            if not opts.dry_run:
                self.client.ln(a, b)
                self.client.rm(a)

        def remove_dir(a):
            if opts.verbose:
                log.info("rmdir %s", a)
            if not opts.dry_run:
                self.client.rm(a)

        retval = None
        if len(paths) < 2:
            log.error("not enough args")
            return 1

        srcs, dst = paths[:-1], paths[-1]
        ndst = unormpath(ujoin(self.cwd, uexpanduser(dst, self.home)))
        try:
            dst_stat = self.client.stat(ndst)
            dst_type = dst_stat["type"]
        except manta.MantaResourceNotFoundError:
            dst_stat = None
            dst_type = None
        if dst_type is not None:
            # TODO: support this eventually
            log.error("target %s exists: this is not yet supported by "
                "the mv comment", dst)
            return 1

        if len(srcs) > 1 and dst_type != "directory":
            log.error("incorrect usage, destination %s is not a directory", dst)
            return 1

        try:
            prepend = (dst_type == "directory") #TODO: use this
            for src in srcs:
                nsrc = unormpath(ujoin(self.cwd, uexpanduser(src, self.home)))
                src_stat = self.client.stat(nsrc)
                if src_stat["type"] != "directory":
                    move_obj(nsrc, ndst)
                else:
                    for dirpath, dirents, objents in self.client.walk(nsrc, False):
                        #pprint((dirpath, dirents, objents))
                        subpath = dirpath[len(nsrc) + 1:]
                        move_dir(dirpath, ujoin(ndst, subpath))
                        for objent in objents:
                            srcpath = ujoin(dirpath, objent["name"])
                            subpath = srcpath[len(nsrc) + 1:]
                            move_obj(srcpath, ujoin(ndst, subpath))
                        remove_dir(dirpath)
        except manta.MantaError:
            _, ex, _ = sys.exc_info()
            log.error(ex)
            return 1

    def _realpath(self, mpath):
        """Normalize the given Manta path and make it absolute
        (relative to cwd).
        """
        return unormpath(ujoin(self.cwd, uexpanduser(mpath, self.home)))

    @cmdln.option("-v", "--verbose", action="store_true",
        help="show link details as it is being linked")
    def do_ln(self, subcmd, opts, object_path, *link_path_args):
        """create a manta link to an object

        Usage:
            ${cmd_name} OBJECT-PATH [LINK-PATH]

        ${cmd_option_list}
        """
        if len(link_path_args) == 1:
            link_path = link_path_args[0]
        elif len(link_path_args) == 0:
            link_path = unormpath(ujoin('.', ubasename(object_path)))
        else:
            raise cmdln.CmdlnUserError("too many arguments: %s %s"
                % (object_path, ' '.join(link_path_args)))
        object_npath = self._realpath(object_path)
        link_npath = self._realpath(link_path)
        if opts.verbose:
            log.info('ln %s %s', object_npath, link_npath)
        self.client.ln(object_npath, link_npath)

    def do_find(self, argv):
        """find paths

        Usage:
            ${cmd_name} [DIRS...] [-name PATTERN] [-type TYPE] [-ls]

        Where "DIRS" are Manta dirs.

        Options:
            -h, --help      show this help message and exit
            -name PATTERN   "PATTERN" is a glob pattern
            -type TYPE      "TYPE" is one of 'd' (for directories),
                            'o' (for objects) or 'f' (for objects).
            -ls             Print the long `ls` output for each entry.
        """
        class Options:
            def __init__(self, *opts):
                for name in opts:
                    setattr(self, name, None)
            def __repr__(self):
                return "<Options: %s>" % dict(
                    (o, getattr(opts, o)) for o in dir(opts)
                      if not o.startswith('__'))

        # Parse DIRS.
        argv.pop(0)  # "find"
        tops = []
        while argv:
            if argv[0].startswith('-'):
                break
            tops.append(argv.pop(0))
        if not tops:
            tops.append('.')
        # Handle globs
        #XXX

        # Parse options.
        opts = Options("name", "type", "ls")
        while argv:
            opt = argv.pop(0)
            if opt == "-name":
                if not argv:
                    log.error("no argument for '-name'")
                    return 1
                opts.name = argv.pop(0)
            elif opt == "-type":
                if not argv:
                    log.error("no argument for '-type'")
                    return 1
                type_char = argv.pop(0)
                opts.type = {
                    "o": "object",
                    "f": "object",
                    "d": "directory",
                }[type_char]
            elif opt == "-ls":
                opts.ls = True
            elif opt in ("-h", "--help"):
                self.do_help(['', 'find'])
                return 0
            else:
                log.error("unknown option: %r", opt)
                return 1
        #print("find: tops=%r, opts=%r" % (tops, opts))

        def find_all(d):
            parent = udirname(d)
            base = ubasename(d)
            dirents = self._get_dir(parent)
            if dirents is None or base not in dirents:
                log.error("%s: no such remote directory", d)
            elif dirents[base]["type"] == "directory":
                for dirpath, dirents, objents in self.client.walk(d):
                    yield {"type": "directory", "name": ubasename(dirpath),
                           "path": dirpath}
                    for objent in sorted(objents, key=itemgetter("name")):
                        objent["path"] = ujoin(dirpath, objent["name"])
                        yield objent
            else:
                objent = dirents[base]
                objent["path"] = d
                yield objent

        # Do the find and filtering.
        for top in tops:
            ntop = self._realpath(top)
            for dirent in find_all(ntop):
                if opts.type and dirent["type"] != opts.type:
                    continue
                if opts.name:
                    base = dirent["name"]
                    if not fnmatchcase(base, opts.name):
                        continue
                path = top + dirent["path"][len(ntop):]
                if opts.ls:
                    self._ls_print_dirent(path, dirent, long=True)
                else:
                    print(path)

    def do_vi(self, subcmd, opts, path):
        """edit a file on Manta (locally in vi)

        Usage:
            ${cmd_name} [OPTIONS...] MANTA-PATH

        ${cmd_option_list}
        Note that this is a limited hack. You need to have a 'vi' to run
        locally. Only one path can be edited. You can edit other files from
        within the Vi session, etc.
        """
        npath = self._realpath(path)
        base = ubasename(npath)
        local_path = os.path.join(CACHE_DIR, "vi",
            "%s-local.%s" % (os.getpid(), base))
        if not os.path.exists(os.path.dirname(local_path)):
            os.makedirs(os.path.dirname(local_path))

        try:
            res, _ = self.client.get_object2(npath, path=local_path)
            is_new = False
            old_digest = res["content-md5"].decode("base64")
            content_type = res["content-type"]
        except manta.MantaAPIError:
            _, ex, _ = sys.exc_info()
            if ex.code == 'ResourceNotFound':
                is_new = True
                content = ""
                content_type = "text/plain"
                old_digest = md5("").digest()
                f = open(local_path, 'w')
                f.write(content)
                f.close()
            else:
                raise

        os.system('vi -f "%s"' % local_path)

        f = open(local_path, 'rb') # TODO: would md5 comparison be preferable?
        new_digest = md5(f.read()).digest()
        f.close()
        if new_digest == old_digest:
            if not is_new:
                log.info("'%s' unchanged.", path)
        else:
            self.client.put(npath, path=local_path, content_type=content_type)
            log.info("'%s' saved.", path)
    do_vi.aliases = ["vim"]

    @cmdln.option("-j", "--json", action="store_true",
        help="display the jobs in JSON")
    @cmdln.option("-s", "--state",
        help="limit to jobs in the given state: 'running', 'done', etc.")
    def do_jobs(self, subcmd, opts):
        """List Manta jobs

        Limitation: at this time `list_jobs` doesn't support paging through
        more than the default response num results.

        Usage:
            ${cmd_name} [OPTIONS...]

        ${cmd_option_list}
        """
        jobs = self.client.list_jobs(state=opts.state)
        if opts.json:
            print(json.dumps(jobs, indent=2))
        else:
            for job in jobs:
                print job['name']

    def do_jobinfo(self, subcmd, opts, job_id):
        """Get details for a Manta job.

        Usage:
            ${cmd_name} [OPTIONS...] JOB-ID

        ${cmd_option_list}
        """
        info = {
            "job": self.client.get_job(job_id),
            "in": self.client.get_job_input(job_id),
            "out": self.client.get_job_output(job_id),
            "err": self.client.get_job_errors(job_id),
            "fail": self.client.get_job_failures(job_id),
        }
        print(json.dumps(info, indent=2))

    def do_job(self, argv):
        """Run a Manta job

        Usage:
            ${cmd_name} [OPTIONS...] [PATHS...] PHASES...

        Phases:
            ^ MAP-PHASE         A map phase command to run.
            ^^ REDUCE-PHASE     A reduce phase command to run.

        Options:
            -h, --help          show this help message and exit
            -v, --verbose       Verbose. Print the job ID (on stderr) and
                                some administrivia for this job.
            -t TIMEOUT, --timeout=TIMEOUT
                                Number of seconds after which to timeout waiting for
                                the job to complete. Default 0 (i.e. never timeout).
            -n NAME, --name NAME
                                Give your job a name. Default's to 'job-$timestamp'
                                for now.

        For the time being, you can use '|' instead of '^'. The latter is
        provided so you don't have to escape '|' from your shell. Back in the
        old school '^' was a synonym for '|' in the shell
        (http://en.wikipedia.org/wiki/Thompson_shell).

        Examples:
            job words.txt ^ grep foo
            job a.log b.log c.log ^ grep bar ^^ wc -l
        """
        #TODO: job `find . -name "email*.txt"` ^ grep foo
        #    Update cmdln.py's line2argv parsing for this? Need a *more*
        #    raw mode for own parsing. What does this line look like *with*
        #    bash in the way?
        #Easier todo this:
        #    find . -name "email*.txt" ^ job -n NAME ^ grep foo
        DEBUG = False

        first_argv = None
        pipe_chars = ['^', '|', '^^']
        pipe_idxs = [argv.index(c) for c in pipe_chars if c in argv]
        if not pipe_idxs:
            if '-h' in argv or '--help' in argv:  # help
                self.do_help(['', 'job'])
                return 0
            pipe_chars_str = "', '".join(pipe_chars[:-1]) \
                + "' or '%s" % pipe_chars[-1]
            raise MantashError("no '%s' pipe char found in job request"
                % pipe_chars_str)
        pipe_idx = min(pipe_idxs)
        first_argv = argv[:pipe_idx]
        rest_argv = argv[pipe_idx:]

        optparser = cmdln.SubCmdOptionParser()
        optparser.set_cmdln_info(self, "job")
        optparser.add_option("-t", "--timeout", type="int", default=0)
        optparser.add_option("-n", "--name")
        optparser.add_option("-v", "--verbose", action="store_true")
        try:
            opts, path_patterns = optparser.parse_args(first_argv[1:])
        except cmdln.StopOptionProcessing:
            return 0
        assert len(path_patterns) > 0, "No KEY paths given"

        # Parse args.
        phases = []
        for arg in rest_argv:
            if arg in ('|', '^'):
                phases.append({"exec": []})
            elif arg == '^^':
                phases.append({"type": "reduce", "exec": []})
            else:
                phases[-1]["exec"].append(arg)
        assert len(phases) > 0, "No job phases given"
        for phase in phases:
            assert len(phase["exec"]) > 0, "Empty job phase"
            # TODO:XXX To double-quote or single-quote? Bash has already
            #   removed the quotes for us. I think we want to single quote.
            #   Is this a "you can't know" question? Lacking info, I think
            #   we want to single quote.
            phase["exec"] = argv2line(phase["exec"])

        keys = []
        for pp in path_patterns:
            try:
                # list of (<is-dir>, <path>, <dirents>)
                paths = self._ls_path(pp, True)
            except MantashError:
                raise MantaError("'%s' does not exist" % pp)
            else:
                for is_dir, p, dirents in paths:
                    for pname, dirent in dirents.items():
                        if dirent["type"] != "object":
                            raise MantashError(
                                "'%s' is a %s (can only process objects)" % (
                                pname, dirent["type"]))
                        keys.append(unormpath(ujoin(self.cwd, pname)))

        if DEBUG:
            print("-- CreateJob")
            print("  keys: %s" % keys)
            print("  phases:\n%s" % _indent(pformat(phases)))
        job_id = self.client.create_job(phases, name=opts.name)
        if DEBUG:
            print("  job_id: %s" % job_id)
        if opts.verbose:
            sys.stderr.write("Created job %s\n" % job_id)
        self.client.add_job_inputs(job_id, keys)
        self.client.end_job_input(job_id)
        #TODO: support cancel_job on ^C
        if opts.verbose:
            sys.stderr.write("Waiting for job %s to complete\n" % job_id)  #TODO log?
        self._wait_for_job(job_id, timeout=opts.timeout)
        outkeys = self.client.get_job_output(job_id)
        for outkey in outkeys:
            log.debug("get job %s output key '%s'", job_id, outkey)
            content = self.client.get(outkey)
            sys.stdout.write(content)
        #XXX Report job failures and errors!

    def _wait_for_job(self, job_id, timeout=30):
        start_time = time.time()
        url = ujoin(self.manta_url, self.account, 'jobs', job_id)
        successive_errors = 0
        #time.sleep(2)
        while True:
            time.sleep(1)
            if timeout and time.time() - start_time >= timeout:
                raise MantashError("job timed out (%ds)" % timeout)
            job = self.client.get_job(job_id)
            #XXX Restore this?
            #if res["status"] != "200":
            #    # TODO: should only allow retry for some error types,
            #    #  e.g. not for 400
            #    successive_errors += 1
            #    if successive_errors >= 3:
            #        raise MantashError("add job keys API error: %s: %s %s: %s" % (
            #            url, res["status"], data["code"], data["message"]))
            #    continue
            successive_errors = 0
            if job["state"] == "done":
                break

    def do_login(self, argv):
        """start a Manta compute login session

        Usage:
            ${cmd_name} [OPTIONS...] [OBJECT]

        Note: This is just calling out to the *node*-manta client tools'
        `mlogin ...`. Thus you need to have those installed and on the PATH.
        """
        # TODO: pass in all args to mlogin using curr manta vars

        # We need to guess which arg, if any, is a manta path and, if so,
        # make it absolute. This could be error prone. I *think* the object
        # must be the last arg for `mlogin`.
        args = argv[1:]
        if args:
            candidate = unormpath(ujoin(self.cwd, args[-1]))
            if self.client.type(candidate) is not None:
                # Last arg is an existing manta path.
                args[-1] = candidate
        opts = [
            '-u', self.manta_url,
            '-k', self.options.key_id,
            '-a', self.account,
        ]
        if self.options.insecure:
            opts += ['-i']
        cmd = ' '.join(['mlogin'] + opts)
        for arg in args:
            cmd += " '%s'" % arg
        log.debug('cmd: %r', cmd)
        subprocess.call(cmd, shell=True)

    def help_login(self):
        try:
            p = subprocess.Popen(['man', 'mlogin'], stdout=subprocess.PIPE)
        except Exception:
            return self.do_login.__doc__
        else:
            stdout, stderr = p.communicate()
            p.wait()
            return self.do_login.__doc__ + '\n\n' + stdout

    def do_sign(self, argv):
        """Produce a signed URL for the given manta path.

        Usage:
            ${cmd_name} [OPTIONS...] OBJECT

        Note: This is just calling out to the *node*-manta client tools'
        `msign ...`. Thus you need to have those installed and on the PATH.
        """
        # TODO: pass in all args to msign using curr manta vars

        # We need to guess which arg, if any, is a manta path and, if so,
        # make it absolute. This could be error prone. I *think* the object
        # must be the last arg for `msign`.
        args = argv[1:]
        if args:
            candidate = unormpath(ujoin(self.cwd, args[-1]))
            if self.client.type(candidate) is not None:
                # Last arg is an existing manta path.
                args[-1] = candidate
        opts = [
            '-u', self.manta_url,
            '-k', self.options.key_id,
            '-a', self.account,
        ]
        if self.options.insecure:
            opts += ['-i']
        cmd = ' '.join(['msign'] + opts)
        for arg in args:
            cmd += " '%s'" % arg
        log.debug('cmd: %r', cmd)
        subprocess.call(cmd, shell=True)

    def help_sign(self):
        try:
            p = subprocess.Popen(['man', 'msign'], stdout=subprocess.PIPE)
        except Exception:
            return self.do_sign.__doc__
        else:
            stdout, stderr = p.communicate()
            p.wait()
            return self.do_sign.__doc__ + '\n\n' + stdout



#---- internal support stuff

def uexpanduser(mpath, home):
    """Like os.path.expanduser for mpaths."""
    if not mpath or not mpath.startswith('~'):
        return mpath
    if mpath == "~":
        return home
    elif mpath[1] != '/':
        return mpath
    else:
        return ujoin(home, mpath[2:])


def argv2line(argv):
    r"""Put together the given argument vector into a command line.

        "argv" is the argument vector to process.

    >>> argv2line(['foo'])
    'foo'
    >>> argv2line(['foo', 'bar'])
    'foo bar'
    >>> argv2line(['foo', 'bar baz'])
    "foo 'bar baz'"
    >>> argv2line(['foo', 'bar "baz"'])
    "foo 'bar \"baz\"'"
    >>> argv2line(['foo"bar'])
    'foo"bar'
    >>> print argv2line(['foo" bar'])
    'foo" bar'
    >>> print argv2line(["foo' bar"])
    "foo' bar"
    >>> argv2line(["foo'bar"])
    "foo'bar"
    """
    escapedArgs = []
    for arg in argv:
        if ' ' in arg or '\t' in arg:
            arg = arg.replace("'", r"\'")
            arg = "'"+arg+"'"
        escapedArgs.append(arg)
    return ' '.join(escapedArgs)


def _indent(s, indent='    '):
    return indent + indent.join(s.splitlines(True))



#---- mainline

def main(argv=sys.argv):
    logging.basicConfig(format='%(name)s: %(levelname)s: %(message)s')
    log.setLevel(logging.INFO)
    shell = Mantash()
    return shell.main(argv, loop=cmdln.LOOP_IF_EMPTY)


## {{{ http://code.activestate.com/recipes/577258/ (r5)
if __name__ == "__main__":
    try:
        retval = main(sys.argv)
    except KeyboardInterrupt:
        sys.exit(1)
    except SystemExit:
        raise
    except:
        import traceback, logging
        if not log.handlers and not logging.root.handlers:
            logging.basicConfig()
        skip_it = False
        exc_info = sys.exc_info()
        if hasattr(exc_info[0], "__name__"):
            exc_class, exc, tb = exc_info
            if isinstance(exc, IOError) and exc.args[0] == 32:
                # Skip 'IOError: [Errno 32] Broken pipe': often a cancelling of `less`.
                skip_it = True
            if not skip_it:
                tb_path, tb_lineno, tb_func = traceback.extract_tb(tb)[-1][:3]
                log.error("%s (%s:%s in %s)", exc_info[1], tb_path,
                    tb_lineno, tb_func)
        else:  # string exception
            log.error(exc_info[0])
        if not skip_it:
            if True or log.isEnabledFor(logging.DEBUG):
                print('')
                traceback.print_exception(*exc_info)
            sys.exit(1)
    else:
        sys.exit(retval)
## end of http://code.activestate.com/recipes/577258/ }}}
