[其他] 使用第三方命令爬取最高人民法院裁判文书

# 配置
	$title = 'class="title|fl print';
	$Web = New-Object System.Net.Webclient;
	$Web.Encoding = [Text.Encoding]::UTF8;
	$url = 'http://www.court.gov.cn/paper/default/index.html';
	$Master = 'http://www.court.gov.cn'
	$http = (split-path $url).Replace('\','/');
	$save = '.\案件记录.log';
	NI $save -type file -force | Out-Null;
#
Function ANJian_ZenLi {
	Param ( [string]$url )
	[Collections.Arraylist] $arr = @();
	[Collections.Arraylist] $array = @();
	$Content = $null;
	$Page = $Web.DownloadString( $url ) -Split "`n";

	$Content = ( $Page | SLS -Pattern $title) -Split "`r`n";
	$array = $Content | %{ $_.Split('"')[-1] -Replace '(</?(li|div))?>','' };
	$array.Add('') | Out-Null;

	$Content = $Page |SLS -Pattern '</div><div style';
	$Content = $Content -Replace ";'>","`r`n@" -Replace '</div>',"`r`n";
	$arr = (($Content -Split "`r`n" | sls -Pattern '^@') -NotMatch '^@<') -Replace '@','';
	$arr = $arr -notmatch '^$'; if ( $arr ) { $arr.Add('') | Out-Null };
	$array = $array + $arr;

	$Content = (( $Page | SLS -Pattern '^\s+.*</div>$') -NotMatch '<div') -Split "`r`n";
	$arr = $Content | %{ ($_ -Replace '^\s+|</div>|&nbsp;','').Trim() };
	$arr = $arr -notmatch '^$'; if ( $arr ) { $arr.Add('') | Out-Null };
	$array = $array + $arr;

	$Content = ( $Page | SLS -Pattern '<(div|p) style.*</(div|p)>$') -Split "`r`n";
	$arr = $Content | %{ $_.Split('"')[-1] -Replace '(</(p|div))?>|&nbsp;','' };
	$arr = $arr -notmatch '^$'; if ( $arr ) { $arr.Add('') | Out-Null };
	$array = $array + $arr;
	$array.Add('*************************完成*************************') | Out-Null;
	Return $array;
};
$Page = $Web.DownloadString( $url ) -Split "`n";
$Last = (( $Page | SLS -Pattern '尾页' ) -Split "`r`n" ).Split('"')[-2];
[int]$Last = $Last.Split('/.')[-2];
[Collections.Arraylist] $PageArray = @($url);
(2..$Last) | %{ $PageArray += $http + '/index/page/' + "$_.html" };

For ( $i =0; $i -lt $PageArray.Count; $i++ ) {
	$DictXianQing = @{};
	if ( $i -ge 1 ) { $Page = $Web.DownloadString( $PageArray[$i] ) -Split "`n"; };
	$Content = ($Page | SLS -Pattern 'xiangqing') -Split "`r`n";

	$Content | %{
		$link = $Master + $_.Split('"')[-2];
		$str = $_.Split('"')[-1] -Replace '(</(a|li))?>','';
		$DictXianQing += @{ "$link" = "$str" };
	};

	Foreach ( $k in $DictXianQing.Keys.GetEnumerator() | Sort {[int]($_.Split('-.')[-2])} ) {
		$Receive = ANJian_ZenLi $k;
		$Receive |ac $save -force;
		''|ac $save -force; ''|ac $save -force;
	};
	'按任意键处理下一页，删除此句会整理所有页面。';pause;
};
'已全部完成 按任意键退出。';pause
复制代码

use Encode;
use Modern::Perl;
use File::Slurp;
use Mojo::UserAgent;
use File::Basename qw/basename/;
use File::Path qw/mkpath/;
STDOUT->autoflush(1);

our $ua = Mojo::UserAgent->new();
our $main = "http://www.court.gov.cn";
our $wdir = "F:/temp/gov_wenshu";
mkpath $wdir unless -e $wdir;

#获取尾页代码，前缀
my ($prefix, $maxpg) = get_max_pgcode( $main ."/wenshu.html" );
for my $id ( 1 .. $maxpg ) {
    printf "${main}${prefix}$id.html\n";
    get_article( "${main}${prefix}$id.html" );
}

sub get_article
{
    our ($main, $wdir);
    my ( $link ) = @_;
    my $res;
    my $fpath;
    my $dom = $ua->get( $link )->result->dom;
    for my $e ( $dom->find(".list .l li a")->each )
    {
        printf "%s\n", basename($e->attr("href"));
        $fpath = $wdir ."/". basename($e->attr("href"));
        next if ( -e $fpath );
        $res = $ua->get( $main . $e->attr("href") )->result;
        write_file( $fpath, $res->body );
    }
}

sub get_max_pgcode
{
    my ( $link ) = @_;
    my $res = $ua->get( $link )->result;
    my $href = $res->dom->at(".yiiPager .last a")->attr("href");
    if ($href =~/^(.*\/)(\d+)\.html/) { return ($1, $2); } 
    else { printf "Failed to get max page code\n"; return undef }
}
复制代码

# 配置
	$title = 'class="title|fl print';
	$Web = New-Object System.Net.Webclient;
	$Web.Encoding = [Text.Encoding]::UTF8;
	$url = 'http://www.court.gov.cn/paper/default/index.html';
	$Master = 'http://www.court.gov.cn'
	$http = (split-path $url).Replace('\','/');
#
Function ANJian_ZenLi {
	Param ( [string]$url )
	[Collections.Arraylist] $arr = @();
	[Collections.Arraylist] $array = @();
	$Content = $null;
	$Page = $Web.DownloadString( $url ) -Split "`n";

	$array = $Page -Match $title | %{ $_.Split('"')[-1] -Replace '(</?(li|div))?>','' };
	$array.Add('') | Out-Null;

	$Content = $Page -Match '</div><div style' -Replace ";'>","`r`n@" -Replace '</div>',"`r`n";
	$arr = ($Content -Split "`n" -Match '^@') -NotMatch '^@<' -Replace '^@','' `
	-Replace '&times;','X' -Replace '&ldquo;','“' -Replace '&rdquo;','”' -Replace '&hellip;','…';
	$arr = $arr -notmatch '^$'; if ( $arr ) { $arr.Add('') | Out-Null };
	$array = $array + $arr;

	$Content = ( $Page -Match '^\s+.*</div>$') -NotMatch '<div';
	$arr = $Content -Replace '^\s+|</div>|&nbsp;','' -Replace '&times;','X' `
	-Replace '&ldquo;','“' -Replace '&rdquo;','”' -Replace '&hellip;','…';
	$arr = $arr -notmatch '^$'; if ( $arr ) { $arr.Add('') | Out-Null };
	$array = $array + $arr;

	$arr = $Page -Match '<(div|p) style.*</(div|p)>$' | %{ $_.Split('"')[-1] -Replace '(</(p|div))?>|&nbsp;','' };
	$arr = $arr -notmatch '^$'; if ( $arr ) { $arr.Add('') | Out-Null };
	$array = $array + $arr;
	$array.Add('*************************完成*************************') | Out-Null;
	'';$array;
};
Function InputLine {
	$num = Read-Host -Prompt "输入内容序号，1 - $n ;按<Enter>+<退格>返回上一层 按2次<Enter>显示内容";
	if ( [Console]::ReadKey($true).Key -eq 'BackSpace' ) { cls;InputPage };
	$num = $num -as [int32];
	if ( !$num -or ($num -lt 1 -or $num -gt $n) ) { CLS;'超出范围,重新输入'; InputLine };
	$Url = $XianQingArray[$num-1][0];cls;
	ANJian_ZenLi $Url;pause;cls;
	Write-Host " 第 $Number 页：如下" -fore Green;
	For ( $k =0; $k -lt $XianQingArray.Count; $k++ ) {
		'';  Write-Host " <$($XianQingArray[$k][1])> " -Fore red -NoNewLine;
		Write-Host $($XianQingArray[$k][2]) -Fore DarkYellow; '';
	};
	InputLine;
};
Function InputPage {
	$Specify = Read-Host -Prompt "输入某一个页面，1 - $Last ";
	[Collections.Arraylist] $XianQingArray = @();
	$Number = $Specify -as [int32];

	if ( !$Number  -or ($Number -lt 1 -or $Number -gt $Last) ) { CLS;'超出范围,重新输入'; InputPage };
	cls; $Page = $Web.DownloadString( $PageArray[$Specify-1] ) -Split "`n";
	$Content = $Page -Match 'xiangqing';

	$n = 0;
	$Content | %{ $n++
		$link = $Master + $_.Split('"')[-2];
		$str = $_.Split('"')[-1] -Replace '(</(a|li))?>','';
		$XianQingArray += ,($link,$n,$str);
	};
	cls; Write-Host " 第 $Number 页：如下" -fore Green;
	For ( $k =0; $k -lt $XianQingArray.Count; $k++ ) {
		'';  Write-Host " <$($XianQingArray[$k][1])> " -Fore red -NoNewLine;
		Write-Host $($XianQingArray[$k][2]) -Fore DarkYellow; '';
	};
	InputLine;
};

$Page = $Web.DownloadString( $url ) -Split "`n";
[int]$Last = ($Page -Match '尾页' -Replace '\D','') -join '';
[Collections.Arraylist] $PageArray = @($url);
(2..$Last) | %{ $PageArray += $http + '/index/page/' + "$_.html" };

InputPage;
复制代码