内存排查手册

硬盘

  • 资源

    df -hT -x tmpfs -x devtmpfs -x squashfs
  • 目录占用

    sudo du -h --max-depth=1 / 2>/dev/null | sort -hr | head -20

内存资源

x99@x99:~$ free -h

               total        used        free      shared  buff/cache   available
内存:          39Gi        30Gi       1.8Gi       1.6Gi       8.8Gi       8.5Gi
交换:         8.0Gi       673Mi       7.3Gi

占用top(静态)

  • 原始信息
x99@x99:~$ ps aux --sort=-%mem | head

USER         PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
x99      3407690 17.4 71.9 86286524 29443164 ?   Sl   09:54   3:00 /www/llama/data/strata/home/.local/share/strata/components/engine/0.1.52-linux-x86_64-cuda-cuda12/strata --serve --pack /www/llama/data/strata/packs/gyro-gyro-m --native /www/llama/data/models_sata/agentionai/Qwen3.8-Flash-Next-Gyro-GGUF/_strata-view/Qwen3.8-Flash-Next-Gyro-M-TQ2_0.gguf --ple-gguf /www/llama/data/models_sata/agentionai/Qwen3.8-Flash-Next-Gyro-GGUF/_strata-view/Qwen3.8-Flash-Next-Gyro-M-TQ2_0.gguf --expert-profile /www/llama/data/strata/build/terminal/data/expert-profile.bin --expert-cache auto --prefill auto --spec 4 --max-context 102400 --kv q4_0 --resident-experts --kv-resident 32768
gdm         2795  0.7  0.6 5039888 250248 tty1   Sl+  9月07 538:26 /usr/bin/gnome-shell
x99      3407663  0.9  0.5 924780 207596 ?       Ssl  09:54   0:09 /www/llama/data/strata/build/terminal/python/bin/python3.12 /www/llama/data/strata/build/terminal/strata.py serve --profile launch-v100-qwen38-125b-strata100k --host 0.0.0.0 --port 8080
root        2025  0.2  0.2 4876100 98004 ?       Ssl  9月07 194:09 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
x99      3016175  0.1  0.1 1440976 66500 ?       Ssl  10月27   1:31 /usr/local/bin/gitea -c /var/lib/gitea/custom/conf/app.ini web
root     3012708  0.5  0.1 1513536 64892 ?       Ssl  10月27   4:27 /mihomo
root         511  0.0  0.1 165836 46696 ?        S<s  9月07   2:24 /usr/lib/systemd/systemd-journald
root     3012758  0.0  0.0 11202208 38712 ?      Ssl  10月27   0:10 node /app/.output/server/index.mjs
root        3191  0.3  0.0 1621248 35996 ?       Sl   9月07 259:45 tailscaled --socket=/tmp/tailscaled.sock --statedir=/var/lib/tailscale --tun=userspace-networking
  • 简化信息
x99@x99:~$ ps -eo pid,user,%mem,rss,comm --sort=-%mem | head -n 11 | awk 'NR==1{printf "%-8s %-10s %-6s %-10s %s\n","PID","USER","%MEM","RSS","COMMAND"; next} {rss=$4; if(rss>=1048576) r=sprintf("%.1fG",rss/1048576); else if(rss>=1024) r=sprintf("%.1fM",rss/1024); else r=rss"K"; printf "%-8s %-10s %-6s %-10s %s\n",$1,$2,$3,r,$5}'

PID      USER       %MEM   RSS        COMMAND
3407690  x99        71.9   28.1G      strata-host
2795     gdm        0.6    244.4M     gnome-shell
3407663  x99        0.5    202.7M     python3.12
2025     root       0.2    95.7M      dockerd
3016175  x99        0.1    64.9M      gitea
3012708  root       0.1    58.9M      mihomo
511      root       0.1    45.6M      systemd-journal
3012758  root       0.0    37.8M      node
3191     root       0.0    35.2M      tailscaled
1600     root       0.0    34.9M      containerd
  • 详细信息
x99@x99:/www/llama$ ps -eo pid,ppid,user,%cpu,%mem,rss,vsz,ni,stat,nlwp,etime,comm --sort=-rss | head -n 11 | awk 'function h(k){if(k>=1048576)return sprintf("%.1fG",k/1048576);if(k>=1024)return sprintf("%.1fM",k/1024);return k"K"} function sw(p, l,a,v){v="-";while((getline l<("/proc/"p"/status"))>0){if(l~/^VmSwap:/){split(l,a,/[ \t]+/);v=a[2];break}}close("/proc/"p"/status");return v} function wc(p, w){w="-";if((getline w<("/proc/"p"/wchan"))>0){if(w=="")w="0"}close("/proc/"p"/wchan");return w} NR==1{printf "%-8s%-8s%-8s%-6s%-6s%-8s%-8s%-8s%-4s%-6s%-6s%-12s%-16s%s\n","PID","PPID","USER","%CPU","%MEM","RSS","VSZ","SWAP","NI","STAT","THR","ELAPSED","COMMAND","WCHAN";next} {s=sw($1);printf "%-8s%-8s%-8s%-6s%-6s%-8s%-8s%-8s%-4s%-6s%-6s%-12s%-16s%s\n",$1,$2,$3,$4,$5,h($6),h($7),(s=="-"?"-":h(s+0)),$8,$9,$10,$11,$12,wc($1)}'

PID     PPID    USER    %CPU  %MEM  RSS     VSZ     SWAP    NI  STAT  THR   ELAPSED     COMMAND         WCHAN
3407690 3407663 x99     4.0   71.9  28.1G   82.3G   0K      0   Sl    49    01:15:47    strata-host     futex_do_wait
2795    2331    gdm     0.7   0.6   246.4M  4.8G    42.6M   0   Sl+   28    50-22:01:49 gnome-shell     0
3407663 3407662 x99     0.4   0.5   202.7M  903.1M  0K      0   Ssl   7     01:15:54    python3.12      hrtimer_nanosleep
2025    1       root    0.2   0.2   95.8M   4.7G    18.0M   0   Ssl   44    50-22:01:50 dockerd         0
3016175 3016051 x99     0.1   0.1   64.9M   1.4G    33.4M   0   Ssl   8     14:38:06    gitea           futex_do_wait
3012708 3012683 root    0.5   0.1   56.5M   1.4G    5.4M    0   Ssl   8     14:42:44    mihomo          0
511     1       root    0.0   0.1   45.8M   161.9M  1.1M    -1  S<s   1     50-22:02:00 systemd-journal 0
3012758 3012733 root    0.0   0.0   37.8M   10.7G   5.6M    0   Ssl   7     14:42:44    node            0
1600    1       root    0.2   0.0   35.3M   4.0G    4.6M    0   Ssl   40    50-22:01:54 containerd      0
3191    2751    root    0.3   0.0   35.2M   1.5G    11.6M   0   Sl    10    50-22:01:48 tailscaled      0

脚本化注入mem

  • .bashrc别名

    chmod +x ~/mem.sh
    vi ~/.bashrc
    mem() { MEM_PROG=mem "$HOME/mem.sh" "$@"; }
  • mem.sh脚本

    #!/usr/bin/env bash
    # ps-mem.sh —— 按物理内存排序的进程速查(命令 mem)
    #   mem            显示帮助(本帮助)
    #   mem list       内存占用 Top10(表头 + 10 行,默认按 RSS 从大到小)
    #   mem N          N<=50:Top N(表头 + N 行)
    #   mem PID        PID>50 的单个数字:只显示该 pid(至多 1 行)
    #   mem a,b,c      全数字逗号分隔:pid in (...)
    #   mem 关键词     字符串:COMMAND/完整命令行包含该词(大小写不敏感)
    #   mem k1,k2..    字符串逗号分隔:包含任一关键词(大小写不敏感)
    #   空格分隔的多个词与逗号等价(mem chrome firefox == mem chrome,firefox)。
    #   -s/--sort K    排序键(只能给一个):mem|men|memory=%MEM↓ cpu=%CPU↓
    #                  rss=RSS↓ vsz=VSZ↓ swap=SWAP↓ thr=THR↓(用量键一律从大到小)
    #                  pid|process=PID↑ time=ELAPSED↓(最久在前) command=COMMAND↑(短名升序)
    #                  不给 -s 默认按 RSS 从大到小;-s 可与上面任一过滤组合
    #                  (例 mem 5 -s cpu / mem chrome -s cpu;光 mem -s cpu = 按 CPU 的 Top10)。
    #   mem kill|stop|terminal|quit pid[,pid...] 结束进程(单个/批量):
    #                  kill=SIGKILL stop=SIGSTOP terminal=SIGTERM quit=SIGQUIT;
    #                  只收 pid,拒 pid 1/自身/祖先链/pid 0,逐 pid 回显 发送/核验。
    # 列(14 列):PID PPID USER %CPU %MEM RSS VSZ SWAP NI STAT THR ELAPSED COMMAND WCHAN
    #   (WCHAN 变长不限宽故垫底;COMMAND=comm,15 字符内核短名)。
    #   ps 只取稳定字段,SWAP(VmSwap)与 WCHAN 用 /proc 补,绕开 procps 字段兼容问题。
    # 管线:ps 取数 → awk 过滤(同时算出 SWAP/WCHAN/秒数,吐 TSV)→ sort 按键排
    #       → top 模式再 head 截 N 行 → awk 格式化(人类可读单位 + 14 列)。
    
    set -uo pipefail
    
    prog="${MEM_PROG:-$(basename "$0")}"
    DEFAULT_TOP=10
    SEP=$'\x1f'
    PSBIN="$(command -v ps)"
    AWKBIN="$(command -v awk)"
    SORTBIN="$(command -v sort)"
    
    usage() {
    cat <<EOF
    用法: $prog [过滤] [-s/--sort 排序键]
      $prog               显示本帮助
      $prog list          内存占用 Top$DEFAULT_TOP(表头 + $DEFAULT_TOP 行,默认按 RSS 从大到小)
      $prog N             N 为 1~50 的数字:Top N(表头 + N 行)
      $prog PID           PID 为 >50 的数字:只显示该 pid(至多 1 行)
      $prog a,b,c         全数字逗号分隔:pid in (a,b,c)
      $prog 关键词        字符串:COMMAND/完整命令行包含该词(大小写不敏感)
      $prog k1,k2,...     字符串逗号分隔:包含任一关键词(大小写不敏感)
      $prog -s K          排序键(只能给一个,可与任一过滤组合;光给 -s = 按该键的 Top$DEFAULT_TOP):
                          mem|men|memory=%MEM↓ cpu=%CPU↓ rss=RSS↓ vsz=VSZ↓ swap=SWAP↓ thr=THR↓
                          pid|process=PID↑ time=ELAPSED↓(最久在前) command=COMMAND↑(短名升序)
      $prog kill pid[,..]     强制杀 SIGKILL(单个或逗号分隔批量)
      $prog stop pid[,..]     暂停 SIGSTOP(恢复:kill -CONT <pid>)
      $prog terminal pid[,..] 终止 SIGTERM(允许清理)
      $prog quit pid[,..]     退出 SIGQUIT(会产生 core)
                          kill 类只接受 pid;拒绝 pid 1 / 自身 / 祖先链 / pid 0;
                          逐 pid 回显 发送/核验 两列
    
    列说明: PID 进程号 | PPID 父进程号 | USER 属主 | %CPU 进程启动至今平均 CPU%
            %MEM 物理内存占比% | RSS 物理内存 | VSZ 虚拟内存 | SWAP 换出量(/proc VmSwap)
            NI nice 值 | STAT 状态 | THR 线程数 | ELAPSED 已运行时间 | COMMAND 内核短名(comm)
            WCHAN 内核等待点(/proc wchan,变长,末列;R 运行态常为 0 或空,属正常)
    示例:
      $prog list  $prog 5  $prog 46117  $prog 46117,24222  $prog chrome  $prog chrome,firefox
      $prog 5 -s cpu  $prog chrome -s cpu  $prog -s swap  $prog list -s time
    EOF
    }
    
    # 0) 先抠出 -s/--sort(只能出现一次、只能带一个值),剩下的走过滤
    sortkey=""
    sortgiven=0
    filterargs=()
    while [ "$#" -gt 0 ]; do
      case "$1" in
        -s|--sort)
          sortgiven=$((sortgiven+1))
          if [ "$#" -lt 2 ]; then echo "$prog: -s/--sort 后面要跟一个排序键" >&2; usage >&2; exit 2; fi
          sortkey="$2"; shift 2;;
        --sort=*)
          sortgiven=$((sortgiven+1))
          sortkey="${1#--sort=}"; shift;;
        -s?*)
          sortgiven=$((sortgiven+1))
          sortkey="${1#-s}"; shift;;
        --)
          shift
          while [ "$#" -gt 0 ]; do filterargs+=("$1"); shift; done;;
        *)
          filterargs+=("$1"); shift;;
      esac
    done
    if [ "$sortgiven" -gt 1 ]; then
      echo "$prog: 排序键只能给一个(多个值排序毫无意义)" >&2; usage >&2; exit 2
    fi
    if [ -n "$sortkey" ]; then
      sortkey="$(printf '%s' "$sortkey" | tr '[:upper:]' '[:lower:]')"
      case "$sortkey" in
        mem|men|memory) sortkey="mem";;
        cpu|rss|vsz|swap|thr|pid|process|time|command) ;;
        *) echo "$prog: 非法排序键 '$sortkey'(允许:mem cpu rss vsz swap thr pid process time command,memory/men=mem)" >&2; usage >&2; exit 2;;
      esac
      [ "$sortkey" = "process" ] && sortkey="pid"
    else
      sortkey="rss"
    fi
    
    # mem kill|stop|terminal|quit pid[,pid...] —— 结束进程(单个或批量)
    do_killverb() {
      local verb="$1"; shift
      local sig
      case "$verb" in
        kill)     sig=KILL;;
        stop)     sig=STOP;;
        terminal) sig=TERM;;
        quit)     sig=QUIT;;
      esac
      if [ "$#" -eq 0 ]; then
        echo "$prog: $verb 后面要跟 pid(单个或逗号分隔,如 $prog $verb 1234 / $prog $verb 1234,5678)" >&2
        usage >&2; return 2
      fi
      local raw t targets=()
      local -A seen=()
      for raw in "$@"; do
        if ! [[ "$raw" =~ ^[0-9]+$ ]]; then
          echo "$prog: $verb 只接受 pid,'$raw' 非法" >&2; usage >&2; return 2
        fi
        t=$((10#$raw))
        if [ "$t" -lt 1 ]; then
          echo "$prog: pid 须 >=1(pid 0 是整组语义,已拒绝)" >&2; usage >&2; return 2
        fi
        if [ -z "${seen[$t]:-}" ]; then seen[$t]=1; targets+=("$t"); fi
      done
      # 保护集:pid 1 + 自身 + 祖先链
      local -A prot=()
      prot[1]=1
      prot[$$]=1
      local myp=$$ pp=""
      while [ "$myp" -gt 1 ]; do
        pp=""
        while read -r k v _; do if [ "$k" = "PPid:" ]; then pp="$v"; break; fi; done < "/proc/$myp/status" 2>/dev/null
        if ! [[ "${pp:-x}" =~ ^[0-9]+$ ]] || [ "$pp" -lt 1 ] || [ "$pp" -ge "$myp" ]; then break; fi
        prot[$pp]=1
        myp=$pp
      done
      local rc=0
      local r_pid=() r_comm=() r_send=() r_check=() sent_idx=()
      local comm err krc i idx st
      for t in "${targets[@]}"; do
        comm="$("$PSBIN" -o comm= -p "$t" 2>/dev/null)"
        comm="${comm//[[:space:]]/}"
        r_pid+=("$t"); r_comm+=("${comm:-?}")
        if [ -n "${prot[$t]:-}" ]; then
          if [ "$t" = "1" ]; then r_send+=("拒绝:pid 1")
          elif [ "$t" = "$$" ]; then r_send+=("拒绝:自身")
          else r_send+=("拒绝:祖先进程"); fi
          r_check+=("-"); rc=1; continue
        fi
        if [ -z "$comm" ]; then r_send+=("失败:不存在"); r_check+=("-"); rc=1; continue; fi
        err="$(builtin kill -s "$sig" -- "$t" 2>&1)"; krc=$?
        if [ $krc -ne 0 ]; then r_send+=("失败:$err"); r_check+=("-"); rc=1; continue; fi
        r_send+=("已发送"); r_check+=("待核验"); sent_idx+=("$(( ${#r_pid[@]} - 1 ))")
      done
      if [ "${#sent_idx[@]}" -gt 0 ]; then
        sleep 0.5
        for idx in "${sent_idx[@]}"; do
          t="${r_pid[$idx]}"
          st="$(cat "/proc/$t/stat" 2>/dev/null)"; st="${st##*) }"; st="${st%% *}"
          if [ "$sig" = "STOP" ]; then
            if [[ "$st" == T* ]]; then r_check[$idx]="已暂停"; else r_check[$idx]="未暂停"; rc=1; fi
          else
            if [ -z "$st" ]; then r_check[$idx]="已退出"
            elif [[ "$st" == Z* ]]; then r_check[$idx]="已退出(僵尸)"
            else r_check[$idx]="仍在运行"; rc=1; fi
          fi
        done
      fi
      printf "%-8s %-16s %-8s %-14s %s\n" "PID" "COMMAND" "SIGNAL" "发送" "核验"
      for i in "${!r_pid[@]}"; do
        printf "%-8s %-16s %-8s %-14s %s\n" "${r_pid[$i]}" "${r_comm[$i]}" "SIG$sig" "${r_send[$i]}" "${r_check[$i]}"
      done
      return $rc
    }
    
    if [ "${#filterargs[@]}" -eq 0 ] && [ "$sortgiven" -eq 0 ]; then
      usage
      exit 0
    fi
    
    # 1) 过滤参数拼起来,按逗号切 token(空格分隔的词也各自成 token)
    input="${filterargs[*]}"
    tokens=()
    rest="$input"
    while :; do
      if [[ "$rest" == *","* ]]; then
        part="${rest%%,*}"
        rest="${rest#*,}"
      else
        part="$rest"
        rest="__DONE__"
      fi
      part="${part#"${part%%[! ]*}"}"
      part="${part%"${part##*[! ]}"}"
      if [ -n "$part" ]; then
        for w in $part; do
          tokens+=("$w")
        done
      fi
      if [ "$rest" = "__DONE__" ]; then break; fi
    done
    
    if [ "${#tokens[@]}" -eq 0 ]; then
      mode="top"; topn="$DEFAULT_TOP"
    else
      lower1="$(printf '%s' "${tokens[0]}" | tr '[:upper:]' '[:lower:]')"
      case "$lower1" in
        kill|stop|terminal|quit)
          if [ "$sortgiven" -gt 0 ]; then echo "$prog: $lower1 不支持 -s/--sort(只接受 pid 列表)" >&2; usage >&2; exit 2; fi
          do_killverb "$lower1" "${tokens[@]:1}"
          exit $?;;
      esac
      if [ "${#tokens[@]}" -eq 1 ] && { [ "$lower1" = "help" ] || [ "$lower1" = "-h" ] || [ "$lower1" = "--help" ]; }; then
        usage
        exit 0
      fi
      if [ "${#tokens[@]}" -eq 1 ] && [ "$lower1" = "list" ]; then
        mode="top"; topn="$DEFAULT_TOP"
      else
        allnum=1
        for t in "${tokens[@]}"; do
          if ! [[ "$t" =~ ^[0-9]+$ ]]; then allnum=0; break; fi
        done
        if [ "$allnum" = "1" ]; then
          if [ "${#tokens[@]}" -eq 1 ]; then
            n="${tokens[0]}"
            if [ "$n" -ge 1 ] 2>/dev/null && [ "$n" -le 50 ]; then
              mode="top"; topn="$n"
            elif [ "$n" -ge 1 ] 2>/dev/null; then
              mode="pids"; pidlist=",$n,"
            else
              echo "$prog: 行数须为正整数(1~50 为 Top N,>50 按 pid 查)" >&2
              usage >&2
              exit 2
            fi
          else
            mode="pids"
            pidlist=",$(IFS=,; printf '%s' "${tokens[*]}"),"
          fi
        else
          mode="keys"
          kws=""
          for t in "${tokens[@]}"; do
            kws="$kws$(printf '%s' "$t" | tr '[:upper:]' '[:lower:]')"$'\x1f'
          done
        fi
      fi
    fi
    
    # 2) 排序键 → sort 参数(TSV 列:1pid 2ppid 3user 4pcpu 5pmem 6rss 7vsz 8ni
    #    9stat 10nlwp 11etime 12comm 13swapnum 14wchan 15esec 16commlc 17args)
    case "$sortkey" in
      mem)     SORTARGS=(-t"$SEP" -k5,5gr -k1,1n);;
      cpu)     SORTARGS=(-t"$SEP" -k4,4gr -k1,1n);;
      rss)     SORTARGS=(-t"$SEP" -k6,6gr -k1,1n);;
      vsz)     SORTARGS=(-t"$SEP" -k7,7gr -k1,1n);;
      swap)    SORTARGS=(-t"$SEP" -k13,13gr -k1,1n);;
      thr)     SORTARGS=(-t"$SEP" -k10,10gr -k1,1n);;
      pid)     SORTARGS=(-t"$SEP" -k1,1n);;
      time)    SORTARGS=(-t"$SEP" -k15,15gr -k1,1n);;
      command) SORTARGS=(-t"$SEP" -k16,16 -k1,1n);;
    esac
    
    PSFIELDS="pid=,ppid=,user=,pcpu=,pmem=,rss=,vsz=,ni=,stat=,nlwp=,etime=,comm=,args="
    
    FILTERLIB='
      function swapnum(pid,    line, a, v) {
        v = -1
        while ((getline line < ("/proc/" pid "/status")) > 0) {
          if (line ~ /^VmSwap:/) { split(line, a, /[ \t]+/); v = a[2]+0; break }
        }
        close("/proc/" pid "/status")
        return v
      }
      function wchanof(pid,    w) {
        w = "-"
        if ((getline w < ("/proc/" pid "/wchan")) > 0) { if (w == "") w = "0" }
        close("/proc/" pid "/wchan")
        gsub(/\x1f/, "?", w)
        return w
      }
      function esec(e,    n, a) {
        n = split(e, a, /[-:]/)
        if (n == 2) return a[1]*60+a[2]
        if (n == 3) return a[1]*3600+a[2]*60+a[3]
        if (n == 4) return a[1]*86400+a[2]*3600+a[3]*60+a[4]
        return 0
      }
      function emit(   s, w) {
        s = $0
        sub(/^ *[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +/, "", s)
        gsub(/\x1f/, "?", s)
        w = wchanof($1)
        printf "%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\n", \
          $1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,swapnum($1),w,esec($11),tolower($12),s
        c++
      }
    '
    FORMATPROG='
      function human(kb) {
        if (kb >= 1048576) return sprintf("%.1fG", kb/1048576)
        if (kb >= 1024)    return sprintf("%.1fM", kb/1024)
        return kb "K"
      }
      BEGIN {
        FS="\x1f"
        printf "%-8s %-8s %-8s %-6s %-6s %-8s %-8s %-8s %-4s %-6s %-6s %-12s %-16s %s\n", \
          "PID","PPID","USER","%CPU","%MEM","RSS","VSZ","SWAP","NI","STAT","THR","ELAPSED","COMMAND","WCHAN"
      }
      {
        c++
        if (lim > 0 && c > lim) next
        sw = ($13 < 0) ? "-" : human($13)
        printf "%-8s %-8s %-8s %-6s %-6s %-8s %-8s %-8s %-4s %-6s %-6s %-12s %-16s %s\n", \
          $1,$2,$3,$4,$5,human($6),human($7),sw,$8,$9,$10,$11,$12,$14
      }
    '
    
    case "${mode:-top}" in
      top)
        "$PSBIN" -eo "$PSFIELDS" --sort=-rss 2>/dev/null \
        | "$AWKBIN" "$FILTERLIB"'
          BEGIN { c=0 }
          { emit() }' \
        | "$SORTBIN" "${SORTARGS[@]}" \
        | "$AWKBIN" -v lim="$topn" "$FORMATPROG"
        ;;
      pids)
        "$PSBIN" -eo "$PSFIELDS" --sort=-rss 2>/dev/null \
        | "$AWKBIN" -v plist="$pidlist" "$FILTERLIB"'
          BEGIN { c=0 }
          { if (index(plist, "," $1 ",") > 0) emit() }
          END { if (c==0) { print "无匹配 pid(无输出行)" > "/dev/stderr"; exit 1 } }' \
        | "$SORTBIN" "${SORTARGS[@]}" \
        | "$AWKBIN" "$FORMATPROG"
        ;;
      keys)
        "$PSBIN" -eo "$PSFIELDS" --sort=-rss 2>/dev/null \
        | "$AWKBIN" -v kws="$kws" "$FILTERLIB"'
          BEGIN { c=0; nk=split(kws, ka, "\x1f") }
          {
            s = $0
            sub(/^ *[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+ +/, "", s)
            lc=tolower($12); la=tolower(s); hit=0
            for (i=1; i<=nk; i++) {
              if (ka[i]=="") continue
              if (index(lc, ka[i])>0 || index(la, ka[i])>0) { hit=1; break }
              if (ka[i] ~ /^[0-9]+$/ && $1==ka[i]) { hit=1; break }
            }
            if (hit) emit()
          }
          END { if (c==0) { print "无匹配关键词(无输出行)" > "/dev/stderr"; exit 1 } }' \
        | "$SORTBIN" "${SORTARGS[@]}" \
        | "$AWKBIN" "$FORMATPROG"
        ;;
    esac