@echo off
setlocal ENABLEDELAYEDEXPANSION
 
set path=%pathgnu%;%path%
echo %path%
 
set sour=新建文件夹
set dest=bbb1
set exclude=QQ
 
del /q "%dest%\*.*"
 
echo %~nx0 %time% 1 >>time.txt
 
for /f "usebackq tokens=* delims=" %%a in (`dir /s /b "%sour%\*.txt"`) do (
    echo "%%~fa"
    type "%%~fa" | sed -r -e "/\S+\s+\S+\s+\S+/^!d" -e "/%exclude%/d" -e "s/SZ/1/" -e "s/SH/0/" -e "s/\s+/^|/g"  >> "%dest%\%%~na.tmp"
)
 
echo,
echo %~nx0 %time% 2 >>time.txt
echo,
 
for /f "usebackq tokens=* delims=" %%a in (`dir /s /b "%dest%\*.tmp"`) do (
    echo "%%~fa"
    type "%%~fa" | sort.exe -u >>"%%~dpna.txt"
)
 
echo %~nx0 %time% 3 >>time.txt
echo,
del %dest%\*.tmpCOPY

@echo off
setlocal ENABLEDELAYEDEXPANSION
 
set path=%pathgnu%;%path%
echo %path%
 
set sour=新建文件夹
set dest=bbb2
set exclude=QQ
 
del /q "%dest%\*.*"
 
echo %~nx0 %time% 1 >>time.txt
 
for /f "usebackq tokens=* delims=" %%a in (`dir /s /b "%sour%\*.txt"`) do (
    echo "%%~fa"
    type "%%~fa" | sed -r -e "/\S+\s+\S+\s+\S+/^!d" -e "/%exclude%/d" -e "s/SZ/1/" -e "s/SH/0/" -e "s/\s+/^|/g"  >> "%dest%\%%~na.tmp"
)
 
echo,
echo %~nx0 %time% 2 >>time.txt
echo,
 
for /f "usebackq tokens=* delims=" %%a in (`dir /s /b "%dest%\*.tmp"`) do (
    echo "%%~fa"
    type "%%~fa" | awk " { arr[$0]++ ; if ( arr[$0] == 1 ) { print $0 } } "  >>"%%~dpna.txt"
)
 
echo %~nx0 %time% 3 >>time.txt
echo,
del %dest%\*.tmpCOPY

Get-ChildItem -path $sour *.txt -Recurse | foreach-object {
    $_.fullname
    $b=( [IO.File]::ReadAllLines($_.fullname) ) -match "\w+\s+\w+\s+[-]?\w+" -notmatch $exclude -replace "SZ","1" -replace "SH","0" -replace "\s+","|"
    $a=$dest,$_.name -join '\'
    [IO.File]::WriteAllLines($a,$b)
}COPY

$sour = "D:\share\tech\New folder\新建文件夹"
$dest = "D:\share\tech\New folder\ccc2"
$exclude = "QQ"
 
Remove-Item $dest\*.*
 
( $MyInvocation.MyCommand.Name + "  1  " + (get-date -Format "HH:mm:ss.ff").tostring() ) | out-file -Encoding ascii -append time.txt
 
Get-ChildItem -path $sour *.txt -Recurse | foreach-object {
    $_.fullname
    $filename = Join-Path -path $dest -ChildPath ($_.basename + ".tmp")
    $a=( [IO.File]::ReadAllLines($_.fullname) )
    $a = $a -match "\S+\s+\S+\s+\S+" -notmatch $exclude -replace "SZ","1" -replace "SH","0" -replace "\s+","|"
    [IO.File]::AppendAllLines([string]$filename , [string[]]$a)
}
 
( $MyInvocation.MyCommand.Name + "  2  " + (get-date -Format "HH:mm:ss.ff").tostring() ) | out-file -Encoding ascii -append time.txt
 
Get-ChildItem -path $dest *.tmp | foreach-object {
    $_.fullname
    $filename = Join-Path -path $dest -ChildPath ($_.basename + ".txt")
    $ht = @{}
    $a = ( [IO.File]::ReadAllLines($_.fullname) )
    $a = $a | foreach-object {
        if ( -not ( $ht.ContainsKey($_) ) )
        {
            $_
            $ht.add($_,"1")
        }
    }
    $a | out-file -Encoding utf8 $filename
}
 
( $MyInvocation.MyCommand.Name + "  3  " + (get-date -Format "HH:mm:ss.ff").tostring() ) | out-file -Encoding ascii -append time.txtCOPY

Get-ChildItem -path $dest *.tmp | foreach-object {
    $reader = New-Object -TypeName System.IO.StreamReader -ArgumentList $_.fullname
    $aa=New-Object System.Collections.Generic.HashSet[string]
    $_.fullname
    $filename = Join-Path -path $dest -ChildPath ($_.basename + ".txt")
    while ( $read = $reader.ReadLine() ) {
        out-null -InputObject $aa.add($read)
    }
    [IO.File]::WriteAllLines($filename,$aa)
}COPY

#&cls&cd /d "%~dp0" & @powershell -c "Get-Content '%~0' | Select-Object -Skip 1 | Out-String | Invoke-Expression" & pause&exit
cls
$t1 = Get-Date
$src_dir = '新建文件夹'
$dst_dir = 'out'
 
[void][System.IO.Directory]::CreateDirectory($dst_dir)
 
#线程函数 处理数据
$HandleGroupJob = {
    #线程参数
    param($dst_dir,$groupInfo)
    Write-Host $groupInfo.Name
    #汇总去重并筛选
	$set = New-Object 'System.Collections.Generic.HashSet[string]'
    & { $groupInfo.Group | foreach { [IO.File]::ReadAllLines($_.FullName)} } | foreach {
		if($_ -match '\S+\s+\d+\s+-?\d{2,}'){
			[void]$set.Add(($_ -replace 'SH','1|' -replace 'SZ','0|' -replace '\s+','|'))
		}
	}
    #输出
    Out-File -InputObject $set -FilePath ('{0}\{1}' -f $dst_dir,$groupInfo.Name) 
	$set = $null
    return ($groupInfo.Name + ' 已完成')
}
 
#多线程设置
$pool = [runspacefactory]::CreateRunspacePool(1,10) #最多10个线程并发
$pool.Open()
$threads = New-Object 'System.Collections.ArrayList'
$results = New-Object 'System.Collections.ArrayList'
 
'开始创建线程...'
Get-ChildItem -Recurse -Path $src_dir -Filter '*.txt' | Group-Object {$_.Name} | foreach {
    $_.Name
    $thread = [powershell]::Create()
    $thread.RunspacePool = $pool
    [void]$thread.AddScript($HandleGroupJob)
    [void]$thread.AddArgument($dst_dir)
    [void]$thread.AddArgument($_)
    [void]$threads.Add($thread)
    [void]$results.Add($thread.BeginInvoke())
}
'-------------------------------'
 
'等待线程结束'
while($true){
    $all_done = $true
    for($i = 0; $i -lt $results.Count; $i++){
        if($results[$i] -ne $null){
            if($results[$i].IsCompleted){
                $threads[$i].EndInvoke($results[$i])
				$threads[$i].Dispose()
				$threads[$i] = $null
                $results[$i] = $null
				[System.GC]::Collect()
            } else {
                $all_done = $false
            }
        }
    }
    if($all_done){ break }
    Start-Sleep -Milliseconds 500
}
 
#关闭线程池
$pool.Close()
'-------------------'
'{0}  -> {1}' -f $t1,(Get-Date)COPY