Source code for cup.shell.oper

#!/usr/bin/python
# -*- coding: utf-8 -*
# Copyright: [CUP] - See LICENSE for details.
# Authors: Zhao Minghao, Guannan Ma
"""
:description:
    shell operations related module
"""

import os
import time
import sys
import shutil
import signal
import random
import hashlib
import platform
import warnings
import datetime
import threading
# import traceback
import subprocess
# import collections


import cup
from cup import decorators
from cup import err


# linux only import
if platform.system() == 'Linux':
    __all__ = [
        'rm', 'rmrf',
        'kill',
        'is_process_used_port', 'is_port_used', 'is_proc_exist',
        'is_proc_exist', 'is_process_running',
        'contains_file', 'backup_file'
    ]
# universal import (platform indepedent)
else:
    __all__ = [
        'contains_file', 'backup_file'
    ]


# linux functionalities {{

# pylint: disable=C0103
def rm(name):
    """
    rm the file if no exception happens.
    Will not raise exception if it fails
    """
    try:
        os.remove(name)
    except OSError as error:
        cup.log.warn("rm oserror: %s" % error)


def rmrf(fpath, safemode=True):
    """
    :param fpath:
        files/direcotry to be deleted.
    :param safemode:
        True by default. You cannot delete root / when safemode is True
    """
    @decorators.needlinux
    def _real_rmrf(fpath, safemode):
        """
        real rmrf
        """
        if safemode:
            if os.path.normpath(os.path.abspath(fpath)) == '/':
                raise err.ShellException('cannot rmtree root / under safemode')
        if os.path.isfile(fpath):
            os.unlink(fpath)
        else:
            shutil.rmtree(fpath)
    return _real_rmrf(fpath, safemode)


def is_process_running(path, name):
    """
    Judge if the executable is running by comparing /proc files.
    :platforms:
        linux only. Will raise exception if running on other platforms
    :param path:
        executable current working direcotry
    :param name:
        executable name
    :return:
        return True if the process is running. Return False otherwise.
    """
    @decorators.needlinux
    def _real_is_proc_exist(path, name):
        """
        _real_is_proc_exist
        """
        path = os.path.realpath(os.path.abspath(path))
        cmd = 'ps -ef|grep %s|grep -v "^grep "|grep -v "^vim "|grep -v "^less "|\
            grep -v "^vi "|grep -v "^cat "|grep -v "^more "|grep -v "^tail "|\
            awk \'{print $2}\'' % (name)
        ret = cup.shell.ShellExec().run(cmd, 10)
        pids = ret['stdout'].strip().split('\n')
        if len(pids) == 0 or len(pids) == 1 and len(pids[0]) == 0:
            return False
        for pid in pids:
            for sel_path in ["cwd", "exe"]:
                cmd = 'ls -l /proc/%s/%s|awk \'{print $11}\' ' % (pid, sel_path)
                ret = cup.shell.ShellExec().run(cmd, 10)
                pid_path = ret['stdout'].strip().strip()
                if pid_path.find(path) == 0:
                    # print '%s is exist: %s' % (name, path)
                    return True
        return False
    return _real_is_proc_exist(path, name)


# for compatibility. Do not delete this line:
is_proc_exist = is_process_running


def _kill_child(pid, sign):
    cmd = 'ps -ef|grep %s|grep -v grep|awk \'{print $2,$3}\'' % (pid)
    ret = cup.shell.ShellExec().run(cmd, 10)
    pids = ret['stdout'].strip().split('\n')
    for proc in pids:
        p_id = proc.split()
        if p_id[1] == pid:
            _kill_child(p_id[0], sign)
        if p_id[0] == pid:
            if len(sign) == 0:
                cup.shell.execshell('kill %s' % pid)
            elif sign == '9' or sign == '-9':
                cup.shell.execshell('kill -9 %s' % pid)
            elif sign == 'SIGSTOP' or sign == '19' or sign == '-19':
                cup.shell.execshell('kill -19 %s' % pid)
            elif sign == 'SIGCONT' or sign == '18' or sign == '-18':
                cup.shell.execshell('kill -18 %s' % pid)
            else:
                cup.log.error('sign error')


def kill(path, name, sign='', b_kill_child=False):
    """
    will judge if the process is running by calling function
    (is_process_running), then send kill signal to this process
    :param path:
        executable current working direcotry (cwd)
    :param name:
        executable name
    :param sign:
        kill sign, e.g. 9 for SIGKILL, 15 for SIGTERM
    :b_kill_child:
        kill child processes or not. False by default.
    """
    path = os.path.realpath(os.path.abspath(path))
    # path = os.path.abspath(path)
    cmd = 'ps -ef|grep %s|grep -v grep|awk \'{print $2}\'' % (name)
    ret = cup.shell.ShellExec().run(cmd, 10)
    pids = ret['stdout'].strip().split('\n')
    for pid in pids:
        cmd = 'ls -l /proc/%s/cwd|awk \'{print $11}\' ' % (pid)
        ret = cup.shell.ShellExec().run(cmd, 10)
        if ret['returncode'] != 0:
            return False
        pid_path = ret['stdout'].strip()
        if pid_path.find(path) == 0 or path.find(pid_path) == 0:
            if b_kill_child is True:
                _kill_child(pid, sign)
            if len(sign) == 0:
                cup.shell.execshell('kill %s' % pid)
            elif sign == '9' or sign == '-9':
                cup.shell.execshell('kill -9 %s' % pid)
            elif sign == 'SIGSTOP' or sign == '19' or sign == '-19':
                cup.shell.execshell('kill -19 %s' % pid)
            elif sign == 'SIGCONT' or sign == '18' or sign == '-18':
                cup.shell.execshell('kill -18 %s' % pid)
            else:
                cup.log.error('sign error')
    return True


[docs]def backup_file(srcpath, filename, dstpath, label=None): """ Backup srcpath/filename to dstpath/filenamne.label. If label is None, cup will use time.strftime('%H:%M:S') :dstpath: will create the folder if no existence """ if label is None: label = time.strftime('%H:%M:%S') if not os.path.exists(dstpath): os.makedirs(dstpath) shutil.copyfile( srcpath + '/' + filename, dstpath + '/' + filename + '.' + label )
def backup_folder(srcpath, foldername, dstpath, label=None): """ same to backup_file except it's a FOLDER not a FILE. """ if label is None: label = time.strftime('%H:%M:%S') if not os.path.exists(dstpath): os.makedirs(dstpath) os.rename( '%s/%s' % (srcpath, foldername), '%s/%s' % (dstpath, foldername + '.' + label) ) def is_path_contain_file(dstpath, dstfile, recursive=False, follow_link=False): """ use contains_file instead. Kept still for compatibility purpose """ return contains_file(dstpath, dstfile, recursive, follow_link)
[docs]def contains_file(dstpath, expected_name, recursive=False, follow_link=False): """ judge if the dstfile is in dstpath :param dstpath: search path :param dstfile: file :param recursive: search recursively or not. False by default. :return: return True on success, False otherwise """ path = os.path.normpath(dstpath) fpath = os.path.normpath(expected_name.strip()) fullpath = '{0}/{1}'.format(path, expected_name.strip()) fullpath = os.path.normpath(fullpath) if recursive: for (_, __, fnames) in os.walk(path, followlinks=follow_link): for filename in fnames: if filename == fpath: return True return False else: if os.path.exists(fullpath): return True else: return False
def is_port_used(port): """ judge if the port is used or not (It's not 100% sure as next second, some other process may steal the port as soon after this function returns) :platform: linux only (netstat command used inside) :param port: expected port :return: return True if the port is used, False otherwise """ @decorators.needlinux def __is_port_used(port): """internal func""" cmd = "netstat -nl | grep ':%s '" % (port) ret = cup.shell.ShellExec().run(cmd, 10) if 0 != ret['returncode']: return False stdout = ret['stdout'].strip() if 0 == len(stdout): return False else: return True return __is_port_used(port) def is_process_used_port(process_path, port): """ judge if a process is using the port :param process_path: process current working direcotry (cwd) :return: Return True if process matches """ # find the pid from by port cmd = "netstat -nlp | grep ':%s '|awk -F ' ' '{print $7}'|\ cut -d \"/\" -f1" % (port) ret = cup.shell.ShellExec().run(cmd, 10) if 0 != ret['returncode']: return False stdout = ret['stdout'].strip() if 0 == len(stdout): return False dst_pid = stdout.strip() # check the path path = os.path.abspath(process_path) for sel_path in ['exe', 'cwd']: cmd = 'ls -l /proc/%s/%s|awk \'{print $11}\' ' % (dst_pid, sel_path) ret = cup.shell.ShellExec().run(cmd, 10) pid_path = ret['stdout'].strip().strip() if 0 == pid_path.find(path): return True return False class Asynccontent(object): """ make a Argcontent to async_run u have to del it after using it """ def __init__(self): self.cmd = None self.timeout = None self.pid = None self.ret = None self.child_list = [] self.__cmdthd = None self.__monitorthd = None self.__subpro = None class ShellExec(object): # pylint: disable=R0903 """ For shell command execution. :: from cup import shell shellexec = shell.ShellExec() # timeout=None will block the execution until it finishes shellexec.run('/bin/ls', timeout=None) # timeout>=0 will open non-blocking mode # The process will be killed if the cmd timeouts shellexec.run(cmd='/bin/ls', timeout=100) """ def __init__(self): self._subpro = None self._subpro_data = None def __kill_process(self, pid): os.kill(pid, signal.SIGKILL) def kill_all_process(self, async_content): """ to kill all process """ for pid in async_content.child_list: self.__kill_process(pid) def get_async_run_status(self, async_content): """ get the command's status """ try: from cup.res import linux async_process = linux.Process(async_content.pid) res = async_process.get_process_status() except err.NoSuchProcess: res = "process is destructor" return res def get_async_run_res(self, async_content): """ if the process is still running the res shoule be None,None,0 """ return async_content.ret def async_run(self, cmd, timeout): """ async_run return a dict {uuid:pid} self.argcontent{cmd,timeout,ret,cmdthd,montor} timeout:returncode:999 cmd is running returncode:-999 """ def _signal_handle(): """ signal setup """ signal.signal(signal.SIGPIPE, signal.SIG_DFL) def _target(argcontent): argcontent.__subpro = subprocess.Popen( argcontent.cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=_signal_handle) from cup.res import linux parent = linux.Process(argcontent.__subpro.pid) children = parent.children(True) ret_dict = [] for process in children: ret_dict.append(process) argcontent.child_list = ret_dict def _monitor(start_time, argcontent): while(int(time.mktime(datetime.datetime.now().timetuple())) - int(start_time) < int(argcontent.timeout)): time.sleep(1) if argcontent.__subpro.poll() is not None: self._subpro_data = argcontent.__subpro.communicate() argcontent.ret['returncode'] = argcontent.__subpro.returncode argcontent.ret['stdout'] = self._subpro_data[0] argcontent.ret['stderr'] = self._subpro_data[1] return str_warn = ( 'Shell "%s"execution timout:%d. To kill it' % (argcontent.cmd, argcontent.timeout) ) argcontent.__subpro.terminate() argcontent.ret['returncode'] = 999 argcontent.ret['stderr'] = str_warn for process in argcontent.child_list: self.__kill_process(process) del argcontent.child_list[:] argcontent = Asynccontent() argcontent.cmd = cmd argcontent.timeout = timeout argcontent.ret = { 'stdout': None, 'stderr': None, 'returncode': -999 } argcontent.__cmdthd = threading.Thread(target=_target, args=(argcontent,)) argcontent.__cmdthd.start() start_time = int(time.mktime(datetime.datetime.now().timetuple())) argcontent.__cmdthd.join(0.1) argcontent.pid = argcontent.__subpro.pid argcontent.__monitorthd = threading.Thread(target=_monitor, args=(start_time, argcontent)) argcontent.__monitorthd.start() #this join should be del if i can make if quicker in Process.children argcontent.__cmdthd.join(0.5) return argcontent def run(self, cmd, timeout): """ refer to the class description :param timeout: If the cmd is not returned after [timeout] seconds, the cmd process will be killed. If timeout is None, will block there until the cmd execution returns :return: returncode == 0 means success, while 999 means timeout { 'stdout' : 'Success', 'stderr' : None, 'returncode' : 0 } E.g. :: import cup shelltool = cup.shell.ShellExec() print shelltool.run('/bin/ls', timeout=1) """ def _signal_handle(): """ signal setup """ signal.signal(signal.SIGPIPE, signal.SIG_DFL) def _target(cmd): self._subpro = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=_signal_handle ) self._subpro_data = self._subpro.communicate() ret = { 'stdout': None, 'stderr': None, 'returncode': 0 } cmdthd = threading.Thread(target=_target, args=(cmd, )) cmdthd.start() cmdthd.join(timeout) if cmdthd.isAlive() is True: str_warn = ( 'Shell "%s"execution timout:%d. To kill it' % (cmd, timeout) ) warnings.warn(str_warn, RuntimeWarning) self._subpro.terminate() ret['returncode'] = 999 ret['stderr'] = str_warn else: self._subpro.wait() times = 0 while self._subpro.returncode is None and times < 10: time.sleep(1) times += 1 ret['returncode'] = self._subpro.returncode assert type(self._subpro_data) == tuple, \ 'self._subpro_data should be a tuple' ret['stdout'] = self._subpro_data[0] ret['stderr'] = self._subpro_data[1] return ret def _do_execshell(cmd, b_printcmd=True, timeout=None): """ do execshell """ if timeout is not None and timeout < 0: raise cup.err.ShellException( 'timeout should be None or >= 0' ) if b_printcmd is True: print 'To exec cmd:%s' % cmd shellexec = ShellExec() return shellexec.run(cmd, timeout) def execshell(cmd, b_printcmd=True, timeout=None): """ 执行shell命令,返回returncode """ return _do_execshell( cmd, b_printcmd=b_printcmd, timeout=timeout)['returncode'] def execshell_withpipe(cmd): """ Deprecated. Use ShellExec instead """ res = os.popen(cmd) return res def execshell_withpipe_ex(cmd, b_printcmd=True): """ Deprecated. Recommand using ShellExec. """ strfile = '/tmp/%s.%d.%d' % ( 'shell_env.py', int(os.getpid()), random.randint(100000, 999999) ) os.mknod(strfile) cmd = cmd + ' 1>' + strfile + ' 2>/dev/null' os.system(cmd) if True == b_printcmd: print cmd fphandle = open(strfile, 'r') lines = fphandle.readlines() fphandle.close() os.unlink(strfile) return lines def execshell_withpipe_str(cmd, b_printcmd=True): """ Deprecated. Recommand using ShellExec. """ return ''.join(execshell_withpipe_ex(cmd, b_printcmd)) def execshell_withpipe_exwitherr(cmd, b_printcmd=True): """ Deprecated. Recommand using ShellExec. """ strfile = '/tmp/%s.%d.%d' % ( 'shell_env.py', int(os.getpid()), random.randint(100000, 999999) ) cmd = cmd + ' >' + strfile cmd = cmd + ' 2>&1' os.system(cmd) if b_printcmd: print cmd fhandle = open(strfile, 'r') lines = fhandle.readlines() fhandle.close() os.unlink(strfile) return lines def is_proc_alive(procname, is_whole_word=False, is_server_tag=False, filters=False): """ Deprecated. Recommand using cup.oper.is_proc_exist """ # print procName if is_whole_word: cmd = "ps -ef|grep -w '%s'$ |grep -v grep" % procname else: cmd = "ps -ef|grep -w '%s' |grep -v grep" % procname if is_server_tag: cmd += '|grep -vwE "vim |less |vi |tail |cat |more "' if filters: if isinstance(filters, str): cmd += "|grep -v '%s'" % filters elif isinstance(filters, list): for _, task in enumerate(filters): cmd += "|grep -v '%s'" % task cmd += '|wc -l' rev = execshell_withpipe_str(cmd, False) if int(rev) > 0: return True else: return False def forkexe_shell(cmd): """ fork a new process to execute cmd (os.system(cmd)) """ try: pid = os.fork() if pid > 0: return except OSError: sys.exit(1) # os.chdir("/") os.setsid() # os.umask(0) try: pid = os.fork() if pid > 0: sys.exit(0) except OSError: sys.exit(1) os.system(cmd) def md5file(filename): """ compute md5 hex value of a file, return with a string (hex-value) """ if os.path.exists(filename) is False: raise IOError('No such file: %s' % filename) with open(filename, 'rb') as fhandle: md5obj = hashlib.md5() while True: strtmp = fhandle.read(131072) # read 128k one time if len(strtmp) <= 0: break md5obj.update(strtmp) return md5obj.hexdigest() def kill9_byname(strname): """ kill -9 process by name """ fd_pid = os.popen("ps -ef | grep -v grep |grep %s \ |awk '{print $2}'" % (strname)) pids = fd_pid.read().strip().split('\n') fd_pid.close() for pid in pids: os.system("kill -9 %s" % (pid)) def kill_byname(strname): """ kill process by name """ fd_pid = os.popen("ps -ef | grep -v grep |grep %s \ |awk '{print $2}'" % (strname)) pids = fd_pid.read().strip().split('\n') fd_pid.close() for pid in pids: os.system("kill -s SIGKILL %s" % (pid)) def del_if_exist(path, safemode=True): """ delete the path if it exists, cannot delete root / under safemode """ if safemode and path == '/': raise IOError('Cannot delete root path /') if os.path.lexists(path) is False: return -1 if os.path.isdir(path): shutil.rmtree(path) elif os.path.isfile(path) or os.path.islink(path): os.unlink(path) else: raise IOError('Does not support deleting the type 4 the path') def rmtree(path, ignore_errors=False, onerror=None, safemode=True): """ safe rmtree. safemode, by default is True, which forbids: 1. not allowing rmtree root "/" """ if safemode: if os.path.normpath(os.path.abspath(path)) == '/': raise err.ShellException('cannot rmtree root / under safemode') if os.path.isfile(path): return os.unlink(path) else: return shutil.rmtree(path, ignore_errors, onerror) def shell_diff(srcfile, dstfile): """ shell diff two files, return 0 if it's the same. """ cmd = 'diff %s %s' % (srcfile, dstfile) return os.system(cmd) def get_pid(process_path, grep_string): """ will return immediately after find the pid which matches 1. ps -ef|grep %s|grep -v grep|grep -vE "^[vim|less|vi|tail|cat|more] " '|awk '{print $2}' 2. workdir is the same as ${process_path} :param process_path: process that runs on :param grep_string: ps -ef|grep ${grep_string} :return: return None if not found. Otherwise, return the pid """ cmd = ( 'ps -ef|grep \'%s\'|grep -v grep|grep -vwE "vim |less |vi |tail |cat |more "' '|awk \'{print $2}\'' ) % (grep_string) ret = cup.shell.ShellExec().run(cmd, 10) pids = ret['stdout'].strip().split('\n') if len(pids) == 0 or len(pids) == 1 and len(pids[0]) == 0: return None for pid in pids: for sel_path in ["cwd", "exe"]: cmd = 'ls -l /proc/%s/%s|awk \'{print $11}\' ' % (pid, sel_path) ret = cup.shell.ShellExec().run(cmd, 10) pid_path = ret['stdout'].strip().strip() if pid_path.find(process_path) == 0: return pid return None # end linux functionalities }} # vi:set tw=0 ts=4 sw=4 nowrap fdm=indent