#!/usr/bin/python
# -*- coding: utf-8 -*
# Copyright: [CUP] - See LICENSE for details.
# Authors: Zhao Minghao, Guannan Ma
"""
:description:
shell operations related module
"""
from __future__ import print_function
import os
import sys
import time
import uuid
import tempfile
import shutil
import signal
import random
import hashlib
import platform
import warnings
import datetime
import threading
import subprocess
import cup
from cup import err
from cup import log
from cup import thread
from cup import platforms
from cup import decorators
# linux only import
if platform.system() == 'Linux':
from cup.res import linux
__all__ = [
'rm', 'rmrf', 'kill',
'is_process_used_port', 'is_port_used', 'is_proc_exist',
'is_proc_exist', 'is_process_running',
'contains_file', 'backup_file',
'ShellExec'
]
# universal import (platform indepedent)
else:
__all__ = [
'contains_file', 'backup_file'
]
# linux functionalities {{
# pylint: disable=C0103
def rm(name):
"""
rm the file if no exception happens.
Will not raise exception if it fails
"""
try:
os.remove(name)
except OSError as error:
cup.log.warn("rm oserror: %s" % error)
def rmrf(fpath, safemode=True):
"""
:param fpath:
files/direcotry to be deleted.
:param safemode:
True by default. You cannot delete root / when safemode is True
"""
@decorators.needlinux
def _real_rmrf(fpath, safemode):
"""
real rmrf
"""
if safemode:
if os.path.normpath(os.path.abspath(fpath)) == '/':
raise err.ShellException('cannot rmtree root / under safemode')
if os.path.isfile(fpath):
os.unlink(fpath)
else:
shutil.rmtree(fpath)
return _real_rmrf(fpath, safemode)
def is_process_running(path, name):
"""
Judge if the executable is running by comparing /proc files.
:platforms:
linux only. Will raise exception if running on other platforms
:param path:
executable current working direcotry
:param name:
executable name
:return:
return True if the process is running. Return False otherwise.
"""
@decorators.needlinux
def _real_is_proc_exist(path, name):
"""
_real_is_proc_exist
"""
path = os.path.realpath(os.path.abspath(path))
cmd = 'ps -ef|grep %s|grep -v "^grep "|grep -v "^vim "|grep -v "^less "|\
grep -v "^vi "|grep -v "^cat "|grep -v "^more "|grep -v "^tail "|\
awk \'{print $2}\'' % (name)
ret = cup.shell.ShellExec().run(cmd, 10)
pids = ret['stdout'].strip().split('\n')
if len(pids) == 0 or len(pids) == 1 and len(pids[0]) == 0:
return False
for pid in pids:
for sel_path in ["cwd", "exe"]:
cmd = 'ls -l /proc/%s/%s|awk \'{print $11}\' ' % (pid, sel_path)
ret = cup.shell.ShellExec().run(cmd, 10)
pid_path = ret['stdout'].strip().strip()
if pid_path.find(path) == 0:
# print('%s is exist: %s' % (name, path))
return True
return False
return _real_is_proc_exist(path, name)
# for compatibility. Do not delete this line:
is_proc_exist = is_process_running
def _kill_child(pid, sign):
cmd = 'ps -ef|grep %s|grep -v grep|awk \'{print $2,$3}\'' % (pid)
ret = cup.shell.ShellExec().run(cmd, 10)
pids = ret['stdout'].strip().split('\n')
for proc in pids:
if len(proc) == 0:
continue
p_id = proc.split()
if p_id[1] == pid:
_kill_child(p_id[0], sign)
if p_id[0] == pid:
if len(sign) == 0:
cup.shell.execshell('kill %s' % pid)
elif sign == '9' or sign == '-9':
cup.shell.execshell('kill -9 %s' % pid)
elif sign == 'SIGSTOP' or sign == '19' or sign == '-19':
cup.shell.execshell('kill -19 %s' % pid)
elif sign == 'SIGCONT' or sign == '18' or sign == '-18':
cup.shell.execshell('kill -18 %s' % pid)
else:
cup.log.error('sign error')
def kill(path, name, sign='', b_kill_child=False):
"""
will judge if the process is running by calling function
(is_process_running), then send kill signal to this process
:param path:
executable current working direcotry (cwd)
:param name:
executable name
:param sign:
kill sign, e.g. 9 for SIGKILL, 15 for SIGTERM
:b_kill_child:
kill child processes or not. False by default.
"""
path = os.path.realpath(os.path.abspath(path))
# path = os.path.abspath(path)
cmd = 'ps -ef|grep %s|grep -v grep|awk \'{print $2}\'' % (name)
ret = cup.shell.ShellExec().run(cmd, 10)
pids = ret['stdout'].strip().split('\n')
for pid in pids:
cmd = 'ls -l /proc/%s/cwd|awk \'{print $11}\' ' % (pid)
ret = cup.shell.ShellExec().run(cmd, 10)
if ret['returncode'] != 0:
return False
pid_path = ret['stdout'].strip()
if pid_path.find(path) == 0 or path.find(pid_path) == 0:
if b_kill_child is True:
_kill_child(pid, sign)
if len(sign) == 0:
cup.shell.execshell('kill %s' % pid)
elif sign == '9' or sign == '-9':
cup.shell.execshell('kill -9 %s' % pid)
elif sign == 'SIGSTOP' or sign == '19' or sign == '-19':
cup.shell.execshell('kill -19 %s' % pid)
elif sign == 'SIGCONT' or sign == '18' or sign == '-18':
cup.shell.execshell('kill -18 %s' % pid)
else:
cup.log.error('sign error')
return True
[docs]def backup_file(srcpath, filename, dstpath, label=None):
"""
Backup srcpath/filename to dstpath/filenamne.label.
If label is None, cup will use time.strftime('%H:%M:S')
:dstpath:
will create the folder if no existence
"""
if label is None:
label = time.strftime('%H:%M:%S')
if not os.path.exists(dstpath):
os.makedirs(dstpath)
shutil.copyfile(
srcpath + '/' + filename, dstpath + '/' + filename + '.' + label
)
def backup_folder(srcpath, foldername, dstpath, label=None):
"""
same to backup_file except it's a FOLDER not a FILE.
"""
if label is None:
label = time.strftime('%H:%M:%S')
if not os.path.exists(dstpath):
os.makedirs(dstpath)
os.rename(
'%s/%s' % (srcpath, foldername),
'%s/%s' % (dstpath, foldername + '.' + label)
)
def is_path_contain_file(dstpath, dstfile, recursive=False, follow_link=False):
"""
use contains_file instead. Kept still for compatibility purpose
"""
return contains_file(dstpath, dstfile, recursive, follow_link)
[docs]def contains_file(dstpath, expected_name, recursive=False, follow_link=False):
"""
judge if the dstfile is in dstpath
:param dstpath:
search path
:param dstfile:
file
:param recursive:
search recursively or not. False by default.
:return:
return True on success, False otherwise
"""
path = os.path.normpath(dstpath)
fpath = os.path.normpath(expected_name.strip())
fullpath = '{0}/{1}'.format(path, expected_name.strip())
fullpath = os.path.normpath(fullpath)
if recursive:
for (_, __, fnames) in os.walk(path, followlinks=follow_link):
for filename in fnames:
if filename == fpath:
return True
return False
else:
if os.path.exists(fullpath):
return True
else:
return False
def is_port_used(port):
"""
judge if the port is used or not (It's not 100% sure as next second, some
other process may steal the port as soon after this function returns)
:platform:
linux only (netstat command used inside)
:param port:
expected port
:return:
return True if the port is used, False otherwise
"""
@decorators.needlinux
def __is_port_used(port):
"""internal func"""
cmd = "netstat -nl | grep ':%s '" % (port)
ret = cup.shell.ShellExec().run(cmd, 10)
if 0 != ret['returncode']:
return False
stdout = ret['stdout'].strip()
if 0 == len(stdout):
return False
else:
return True
return __is_port_used(port)
def is_process_used_port(process_path, port):
"""
judge if a process is using the port
:param process_path:
process current working direcotry (cwd)
:return:
Return True if process matches
"""
# find the pid from by port
cmd = "netstat -nlp | grep ':%s '|awk -F ' ' '{print $7}'|\
cut -d \"/\" -f1" % (port)
ret = cup.shell.ShellExec().run(cmd, 10)
if 0 != ret['returncode']:
return False
stdout = ret['stdout'].strip()
if 0 == len(stdout):
return False
dst_pid = stdout.strip()
# check the path
path = os.path.abspath(process_path)
for sel_path in ['exe', 'cwd']:
cmd = 'ls -l /proc/%s/%s|awk \'{print $11}\' ' % (dst_pid, sel_path)
ret = cup.shell.ShellExec().run(cmd, 10)
pid_path = ret['stdout'].strip().strip()
if 0 == pid_path.find(path):
return True
return False
class Asynccontent(object):
"""
make a Argcontent to async_run u have to del it after using it
"""
def __init__(self):
self.cmd = None
self.timeout = None
self.pid = None
self.ret = {
'stdout': None,
'stderr': None,
'returncode': 0
}
self.child_list = []
self.cmdthd = None
self.monitorthd = None
self.subproc = None
self.tempscript = None
class ShellExec(object): # pylint: disable=R0903
"""
For shell command execution.
::
from cup import shell
shellexec = shell.ShellExec()
# timeout=None will block the execution until it finishes
shellexec.run('/bin/ls', timeout=None)
# timeout>=0 will open non-blocking mode
# The process will be killed if the cmd timeouts
shellexec.run(cmd='/bin/ls', timeout=100)
"""
def __init__(self, tmpdir='/tmp/'):
"""
:param tmpdir:
shellexec will use tmpdir to handle temp files
"""
self._subpro = None
self._subpro_data = None
self._tmpdir = tmpdir
self._tmpprefix = 'cup.shell.{0}'.format(uuid.uuid4())
@classmethod
def kill_all_process(cls, async_content):
"""
to kill all process
"""
for pid in async_content.child_list:
os.kill(pid, signal.SIGKILL)
@classmethod
def which(cls, pgm):
"""get executable"""
if os.path.exists(pgm) and os.access(pgm, os.X_OK):
return pgm
path = os.getenv('PATH')
for fpath in path.split(os.path.pathsep):
fpath = os.path.join(fpath, pgm)
if os.path.exists(fpath) and os.access(fpath, os.X_OK):
return fpath
@classmethod
def get_async_run_status(cls, async_content):
"""
get the process status of executing async cmd
:return:
None if the process has finished.
Otherwise, return a object of linux.Process(async_pid)
"""
try:
async_process = linux.Process(async_content.pid)
res = async_process.get_process_status()
except err.NoSuchProcess:
res = None
return res
@classmethod
def get_async_run_res(cls, async_content):
"""
if the process is still running the res shoule be None,None,0
"""
return async_content.ret
def async_run(self, cmd, timeout):
"""
async_run
return a dict {uuid:pid}
self.argcontent{cmd,timeout,ret,cmdthd,montor}
timeout:returncode:999
cmd is running returncode:-999
"""
def _signal_handle():
"""
signal setup
"""
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
def _target(argcontent, proc_cond):
argcontent.tempscript = tempfile.NamedTemporaryFile(
dir=self._tmpdir, prefix=self._tmpprefix,
delete=True
)
with open(argcontent.tempscript.name, 'w+b') as fhandle:
fhandle.write('cd {0};\n'.format(os.getcwd()))
fhandle.write(argcontent.cmd)
shexe = self.which('sh')
cmds = [shexe, argcontent.tempscript.name]
log.info(
'to async execute {0} with script {1}'.format(
argcontent.cmd, cmds)
)
try:
proc_cond.acquire()
argcontent.subproc = subprocess.Popen(
cmds, stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=_signal_handle)
proc_cond.notify()
proc_cond.release()
except OSError:
proc_cond.notify()
proc_cond.release()
argcontent.ret['returncode'] = -1
argcontent.ret['stderr'] = (
'failed to execute the cmd, plz check it out\'s'
)
def _monitor(start_time, argcontent):
while(int(time.mktime(datetime.datetime.now().timetuple())) - int(start_time) <
int(argcontent.timeout)):
time.sleep(1)
if argcontent.subproc.poll() is not None:
self._subpro_data = argcontent.subproc.communicate()
argcontent.ret['returncode'] = argcontent.subproc.returncode
argcontent.ret['stdout'] = self._subpro_data[0]
argcontent.ret['stderr'] = self._subpro_data[1]
return
parent = linux.Process(argcontent.subproc.pid)
children = parent.children(True)
ret_dict = []
for process in children:
ret_dict.append(process)
argcontent.child_list = ret_dict
str_warn = (
'Shell "{0}"execution timout:{1}. To kill it'.format(
argcontent.cmd, argcontent.timeout)
)
self.kill_all_process(argcontent)
argcontent.ret['returncode'] = 999
argcontent.ret['stderr'] = str_warn
argcontent.subproc.terminate()
argcontent = Asynccontent()
argcontent.cmd = cmd
argcontent.timeout = timeout
argcontent.ret = {
'stdout': None,
'stderr': None,
'returncode': -999
}
proc_cond = threading.Condition(threading.Lock())
argcontent.cmdthd = threading.Thread(
target=_target, args=(argcontent, proc_cond))
argcontent.cmdthd.daemon = True
proc_cond.acquire()
argcontent.cmdthd.start()
start_time = int(time.mktime(datetime.datetime.now().timetuple()))
argcontent.cmdthd.join(0.1)
proc_cond.wait()
proc_cond.release()
if argcontent.subproc is not None:
argcontent.pid = argcontent.subproc.pid
argcontent.monitorthd = threading.Thread(target=_monitor,
args=(start_time, argcontent))
argcontent.monitorthd.daemon = True
argcontent.monitorthd.start()
#this join should be del if i can make if quicker in Process.children
argcontent.cmdthd.join(0.5)
return argcontent
def run(self, cmd, timeout):
"""
refer to the class description
:param timeout:
If the cmd is not returned after [timeout] seconds, the cmd process
will be killed. If timeout is None, will block there until the cmd
execution returns
:return:
{
'stdout' : 'Success',
'stderr' : None,
'returncode' : 0
}
returncode == 0 means success, while 999 means timeout
E.g.
::
import cup
shelltool = cup.shell.ShellExec()
print shelltool.run('/bin/ls', timeout=1)
"""
def _signal_handle():
"""
signal setup
"""
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
def _trans_bytes(data):
"""trans bytes into unicode for python3"""
if platforms.is_py2():
return data
if isinstance(data, bytes):
try:
data = bytes.decode(data)
except Exception:
data = 'Error to decode result'
return data
def _pipe_asshell(cmd):
"""
run shell with subprocess.Popen
"""
tempscript = tempfile.NamedTemporaryFile(
dir=self._tmpdir, prefix=self._tmpprefix,
delete=True
)
with open(tempscript.name, 'w+') as fhandle:
fhandle.write('cd {0};\n'.format(os.getcwd()))
fhandle.write(cmd)
shexe = self.which('sh')
cmds = [shexe, tempscript.name]
log.info(
'cup shell execute {0} with script {1}'.format(
cmd, cmds)
)
self._subpro = subprocess.Popen(
cmds, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, preexec_fn=_signal_handle
)
self._subpro_data = self._subpro.communicate()
ret = {
'stdout': None,
'stderr': None,
'returncode': 0
}
cmdthd = threading.Thread(
target=_pipe_asshell, args=(cmd, )
)
cmdthd.start()
cmdthd.join(timeout)
if thread.thread_alive(cmdthd):
str_warn = (
'Shell "%s"execution timout:%d. Killed it' % (cmd, timeout)
)
warnings.warn(str_warn, RuntimeWarning)
parent = linux.Process(self._subpro.pid)
for child in parent.children(True):
os.kill(child, signal.SIGKILL)
ret['returncode'] = 999
ret['stderr'] = str_warn
self._subpro.terminate()
else:
self._subpro.wait()
times = 0
while self._subpro.returncode is None and times < 10:
time.sleep(1)
times += 1
ret['returncode'] = self._subpro.returncode
assert type(self._subpro_data) == tuple, \
'self._subpro_data should be a tuple'
ret['stdout'] = _trans_bytes(self._subpro_data[0])
ret['stderr'] = _trans_bytes(self._subpro_data[1])
return ret
def _do_execshell(cmd, b_printcmd=True, timeout=None):
"""
do execshell
"""
if timeout is not None and timeout < 0:
raise cup.err.ShellException(
'timeout should be None or >= 0'
)
if b_printcmd is True:
print('To exec cmd:{0}'.format(cmd))
shellexec = ShellExec()
return shellexec.run(cmd, timeout)
def execshell(cmd, b_printcmd=True, timeout=None):
"""
执行shell命令，返回returncode
"""
return _do_execshell(
cmd, b_printcmd=b_printcmd, timeout=timeout)['returncode']
def execshell_withpipe(cmd):
"""
Deprecated. Use ShellExec instead
"""
res = os.popen(cmd)
return res
def execshell_withpipe_ex(cmd, b_printcmd=True):
"""
Deprecated. Recommand using ShellExec.
"""
strfile = '/tmp/%s.%d.%d' % (
'shell_env.py', int(os.getpid()), random.randint(100000, 999999)
)
os.mknod(strfile)
cmd = cmd + ' 1>' + strfile + ' 2>/dev/null'
os.system(cmd)
if True == b_printcmd:
print(cmd)
fphandle = open(strfile, 'r')
lines = fphandle.readlines()
fphandle.close()
os.unlink(strfile)
return lines
def execshell_withpipe_str(cmd, b_printcmd=True):
"""
Deprecated. Recommand using ShellExec.
"""
return ''.join(execshell_withpipe_ex(cmd, b_printcmd))
def execshell_withpipe_exwitherr(cmd, b_printcmd=True):
"""
Deprecated. Recommand using ShellExec.
"""
strfile = '/tmp/%s.%d.%d' % (
'shell_env.py', int(os.getpid()), random.randint(100000, 999999)
)
cmd = cmd + ' >' + strfile
cmd = cmd + ' 2>&1'
os.system(cmd)
if b_printcmd:
print(cmd)
fhandle = open(strfile, 'r')
lines = fhandle.readlines()
fhandle.close()
os.unlink(strfile)
return lines
def is_proc_alive(procname, is_whole_word=False, is_server_tag=False, filters=False):
"""
Deprecated. Recommand using cup.oper.is_proc_exist
"""
# print procName
if is_whole_word:
cmd = "ps -ef|grep -w '%s'$ |grep -v grep" % procname
else:
cmd = "ps -ef|grep -w '%s' |grep -v grep" % procname
if is_server_tag:
cmd += '|grep -vwE "vim |less |vi |tail |cat |more "'
if filters:
if isinstance(filters, str):
cmd += "|grep -v '%s'" % filters
elif isinstance(filters, list):
for _, task in enumerate(filters):
cmd += "|grep -v '%s'" % task
cmd += '|wc -l'
rev = execshell_withpipe_str(cmd, False)
if int(rev) > 0:
return True
else:
return False
def forkexe_shell(cmd):
"""
fork a new process to execute cmd (os.system(cmd))
"""
try:
pid = os.fork()
if pid > 0:
return
except OSError:
sys.exit(1)
# os.chdir("/")
os.setsid()
# os.umask(0)
try:
pid = os.fork()
if pid > 0:
sys.exit(0)
except OSError:
sys.exit(1)
os.system(cmd)
def md5file(filename):
"""
compute md5 hex value of a file, return with a string (hex-value)
"""
if os.path.exists(filename) is False:
raise IOError('No such file: %s' % filename)
with open(filename, 'rb') as fhandle:
md5obj = hashlib.md5()
while True:
strtmp = fhandle.read(131072) # read 128k one time
if len(strtmp) <= 0:
break
if isinstance(strtmp, unicode):
md5obj.update(strtmp.encode('utf-8'))
else:
md5obj.update(strtmp)
return md5obj.hexdigest()
def kill9_byname(strname):
"""
kill -9 process by name
"""
fd_pid = os.popen("ps -ef | grep -v grep |grep %s \
|awk '{print $2}'" % (strname))
pids = fd_pid.read().strip().split('\n')
fd_pid.close()
for pid in pids:
os.system("kill -9 %s" % (pid))
def kill_byname(strname):
"""
kill process by name
"""
fd_pid = os.popen("ps -ef | grep -v grep |grep %s \
|awk '{print $2}'" % (strname))
pids = fd_pid.read().strip().split('\n')
fd_pid.close()
for pid in pids:
os.system("kill -s SIGKILL %s" % (pid))
def del_if_exist(path, safemode=True):
"""
delete the path if it exists, cannot delete root / under safemode
"""
if safemode and path == '/':
raise IOError('Cannot delete root path /')
if os.path.lexists(path) is False:
return -1
if os.path.isdir(path):
shutil.rmtree(path)
elif os.path.isfile(path) or os.path.islink(path):
os.unlink(path)
else:
raise IOError('Does not support deleting the type 4 the path')
def rmtree(path, ignore_errors=False, onerror=None, safemode=True):
"""
safe rmtree.
safemode, by default is True, which forbids:
1. not allowing rmtree root "/"
"""
if safemode:
if os.path.normpath(os.path.abspath(path)) == '/':
raise err.ShellException('cannot rmtree root / under safemode')
if os.path.isfile(path):
return os.unlink(path)
else:
return shutil.rmtree(path, ignore_errors, onerror)
def shell_diff(srcfile, dstfile):
"""
shell diff two files, return 0 if it's the same.
"""
cmd = 'diff %s %s' % (srcfile, dstfile)
return os.system(cmd)
def get_pid(process_path, grep_string):
"""
will return immediately after find the pid which matches
1. ps -ef|grep %s|grep -v grep|grep -vE "^[vim|less|vi|tail|cat|more] "
'|awk '{print $2}'
2. workdir is the same as ${process_path}
:param process_path:
process that runs on
:param grep_string:
ps -ef|grep ${grep_string}
:return:
return None if not found. Otherwise, return the pid
"""
cmd = (
'ps -ef|grep \'%s\'|grep -v grep|grep -vwE "vim |less |vi |tail |cat |more "'
'|awk \'{print $2}\''
) % (grep_string)
ret = cup.shell.ShellExec().run(cmd, 10)
pids = ret['stdout'].strip().split('\n')
if len(pids) == 0 or len(pids) == 1 and len(pids[0]) == 0:
return None
for pid in pids:
for sel_path in ["cwd", "exe"]:
cmd = 'ls -l /proc/%s/%s|awk \'{print $11}\' ' % (pid, sel_path)
ret = cup.shell.ShellExec().run(cmd, 10)
pid_path = ret['stdout'].strip().strip()
if pid_path.find(process_path) == 0:
return pid
return None
# end linux functionalities }}
# vi:set tw=0 ts=4 sw=4 nowrap fdm=indent