批处理之家 - Powered by Discuz! Board

@set @n=0;/*&echo off
cscript /nologo /e:javascript %0 <输入.txt >输出.txt
pause
exit/b
*/
var t = new Date().getTime();
var i,j,m,n
var count,bingo,rate,length
var debug = false //调试选项
var format = true //是否将过滤文本以提高准确度，如启用，下文参数皆以格式化后为准
var step = 4 //分段长度数
var text = WScript.StdIn.ReadAll()
text = text.replace(/(?:\r?\n)+/mg,'\n')
var ar = text.split(/\n/g)
//分配对象属性
for(i in ar){
ar[i] = {str:ar[i],ar:ar[i]}
if (format) { //过滤文本
ar[i].ar = ar[i].ar.replace(/[—：、；，。！？…～―（）｛｝《》\x00-\x09\x0b\x0c\x0e-\x7f]+/g,' ') //过滤标点
ar[i].ar = ar[i].ar.replace(/(?:^| )[一二三四五六七八九十百]+(?:$| )/g,'') //过滤数词
ar[i].ar = ar[i].ar.replace(/[又很太着了的地得不]/g,'') //过滤常见副词
ar[i].ar = ar[i].ar.replace(/ +/g,' ') //压缩重复符号
ar[i].ar = ar[i].ar.replace(/^ | $/g,'') //过滤首尾符号
}
}
//过滤被包含行
for(i=ar.length-1;i>0;i--){
for(m=i-1;m>=0;m--){
if(ar[m].ar.indexOf(ar[i].ar)!=-1||ar[i].ar.indexOf(ar[m].ar)!=-1){
if(ar[i].ar.length>ar[m].ar.length)ar[m]=ar[i]
ar.splice(i,1)
break
}
}
}
//行分段
for(i in ar){
ar[i].ar = (
str2ar(ar[i].ar,0,step).concat(
str2ar(ar[i].ar,1,step),
str2ar(ar[i].ar,2,step),
str2ar(ar[i].ar,3,step)
)
)
}
for(i=ar.length-1;i>0;i--){
for(m=i-1;m>=0;m--){
bingo=bingo_max=count=0
for(j in ar[i].ar){
for(n in ar[m].ar){
if(ar[i].ar[j]===ar[m].ar[n]){
bingo++
} else {
count+=bingo
bingo_max = Math.max(bingo_max,bingo)
bingo = 0
}
}
}
length = Math.max(ar[i].ar.length,ar[m].ar.length)
//过滤相似行
count+=bingo
if (count/length > Math.sin(2/(Math.sqrt(length+5)))) {
if(ar[i].ar.length==ar[m].ar.length)ar[m]=ar[i]
ar.splice(i,1)
break
}
//过滤连续相似行
bingo_max = Math.max(bingo_max,bingo)
if (bingo_max/length > 1/Math.sqrt(length+1)) {
if(ar[i].ar.length==ar[m].ar.length)ar[m]=ar[i]
ar.splice(i,1)
break
}
}
}
for(i in ar){
WScript.Echo(ar[i].str)
}
if(debug)WScript.Echo(new Date().getTime()-t)
function str2ar(str,i,step){
var ar = new Array()
for(;i<str.length;i+=step){
ar.push(str.substr(i,step))
}
return ar
}

复制代码

Const input = "输入.txt"
Const output = "输出.txt"
Const isDebug = False
Const length = 4
t=Timer
Set fso = CreateObject("Scripting.FileSystemObject")
Set ts = fso.OpenTextFile(input,1)
ar = Split(ts.ReadAll,vbCrLf)
For i=UBound(ar) To 1 Step -1
If Not test(ar,i) Then ar(i)=""
Next
Set ts = fso.CreateTextFile(output,1)
For Each a In ar
If Len(a) Then ts.WriteLine a
Next
If isDebug Then ts.WriteLine timer-t
Function test(ar,i)
Dim j,length
For j=0 To i-1
length = Len(ar(i))
If Len(ar(j))>Len(ar(i)) Then length = Len(ar(j))
If GetLevenshteinDistince(ar(i), ar(j)) > Sin(2/(Sqr(length)+5)) Then
test = False
Exit Function
End If
Next
test = True
End Function
Function GetLevenshteinDistince(str1, str2) '函数引用自：http://bbs.bathome.net/thread-27991-1-1.html
Dim x, y, A, B, C, K
Dim l1,l2
Dim Matrix()
l1 = l2 = 0
If Len(str2)>=length Then l1=Len(str2)-length+1
If Len(str1)>=length Then l2=Len(str1)-length+1
ReDim Matrix(l2, l1)
'初始化第一行和第一列
For x = 0 To UBound(Matrix, 1)
Matrix(x, 0) = x
Next
For y = 0 To UBound(Matrix, 2)
Matrix(0, y) = y
Next
'填充矩阵
For x = 1 To UBound(Matrix, 1)
For y = 1 To UBound(Matrix, 2)
If (Mid(str1, Matrix(0, y), 4) = Mid(str2, Matrix(x, 0), 4)) Then
C = Matrix(x -1 ,y - 1)
Else
C = Matrix(x -1 ,y - 1) + 1
End If
A = Matrix(x - 1, y) + 1
B = Matrix(x, y - 1) + 1
If (A =< B and A =< C) Then Matrix(x, y) = A
If (B =< C and B =< A) Then Matrix(x, y) = B
If (C =< A and C =< B) Then Matrix(x, y) = C
Next
Next
'计算 LD 值
If (Len(str1) > Len(str2)) Then
K = Len(str1)-length+1
Else
K = Len(str2)-length+1
End If
GetLevenshteinDistince = 1 - (Matrix(l2, l1) / K)
End Function

复制代码

1、剔除完全被包含的行
2、剔除相似度超过阀值的行（算法：count/length = sin(2/(sqrt(length+5)))）
3、剔除连续相似次数超过阀值的行（算法：count/length = 1/sqrt(length+1)）

复制代码