| 両方とも前のリビジョン 前のリビジョン | |
| mitosearchデータ検証 [2022/09/14 06:43] – [陸か水かどうか] 133.11.50.163 | mitosearchデータ検証 [Unknown date] (現在) – 削除 - 外部編集 (Unknown date) 127.0.0.1 |
|---|
| ## 東京湾の魚の月平均表 | |
| |
| ``` | |
| for i in `more data/fish/lat-long-date.txt |grep N|grep E|awk '$2>35&&$2<35.69 && $4>139.6&&$4<140.2'|cut -f 1`; do cat db_fish/$i.input; done|awk -F'\t' '$0~"^20"{split($1,arr,"-"); m=arr[2]; n[m]++} $0!~"^20"{d[$1][m]+=$2} END{for(i in d){max=0; for(j in d[i]){d[i][j]=d[i][j]/n[m]; if(d[i][j]>max){max=d[i][j]}}; print max"\t"i"\t"d[i]["01"]"\t"d[i]["02"]"\t"d[i]["03"]"\t"d[i]["04"]"\t"d[i]["05"]"\t"d[i]["06"]"\t"d[i]["07"]"\t"d[i]["08"]"\t"d[i]["09"]"\t"d[i]["10"]"\t"d[i]["11"]"\t"d[i]["12"]}}'|sort -k1,1nr|awk -F'\t' '{ORS=""; print $1"\t"$2; for(i=3;i<=14;i++){print "\t"$i/$1*100}; print "\n"}' > /tmp/aa | |
| ``` | |
| |
| ## 陸か水かどうか | |
| |
| ``` | |
| [yoshitake@m50v251n3 yoshitake]$ more mapwater.py | |
| import sys | |
| from staticmap import StaticMap | |
| map = StaticMap(50, 50, url_template='http://a.tile.openstreetmap.org/{z}/{x}/{y}.png') | |
| img = map.render(zoom=17, center=[float(sys.argv[1]), float(sys.argv[2])]) | |
| pixelSizeTuple = img.size | |
| water = 0 | |
| for i in range(pixelSizeTuple[0]): | |
| for j in range(pixelSizeTuple[1]): | |
| r,g,b = img.getpixel((i,j)) | |
| if (r == 170 and g == 211 and b == 223): | |
| water = 1 | |
| |
| print(water) | |
| |
| [yoshitake@m50v251n3 yoshitake]$ pwd | |
| /home/yoshitake/yoshitake | |
| ``` | |
| |
| ``` | |
| more ~/mitosearch/Mitosearch/data/fish/lat-long-date.txt |awk -F'\t' '$2~"^[0-9]"{split($2,arr," "); if(arr[2]=="N"){k=arr[1]}else{k=-arr[1]}; if(arr[4]=="E"){k2=arr[3]}else{k2=-arr[3]}; print $1"\t"k"\t"k2}'|awk '{print "echo "$1"; docker run -v $PWD:$PWD -w $PWD -i --rm python3-staticmap python mapwater.py "$3" "$2}'|xargs -I{} bash -c "{}" > mapwater.result | |
| |
| cat mapwater.result|paste - - > mapwater.result.txt | |
| ``` | |
| |
| ``` | |
| import sys | |
| from staticmap import StaticMap | |
| baser=[150,129,142,135,123] | |
| baseg=[200,170,189,179,162] | |
| baseb=[219,182,205,193,172] | |
| map = StaticMap(50, 50) #, url_template='http://a.tile.openstreetmap.org/{z}/{x}/{y}.png') | |
| img = map.render(zoom=17, center=[float(sys.argv[1]), float(sys.argv[2])]) | |
| pixelSizeTuple = img.size | |
| water = 0 | |
| minsum = 255*255*3 | |
| minr=0 | |
| ming=0 | |
| minb=0 | |
| for i in range(pixelSizeTuple[0]): | |
| for j in range(pixelSizeTuple[1]): | |
| r,g,b = img.getpixel((i,j)) | |
| for k in range(len(baser)): | |
| tempsum = (r-baser[k])*(r-baser[k])+(g-baseg[k])*(g-baseg[k])+(b-baseb[k])*(b-baseb[k]) | |
| if (tempsum == 0): | |
| water = 1 | |
| elif tempsum < minsum: | |
| minsum = tempsum | |
| minr=r | |
| ming=g | |
| minb=b | |
| |
| if water == 1: | |
| print(water) | |
| else: | |
| print(str(water)+" "+str(minsum)+" "+str(minr)+","+str(ming)+","+str(minb)) | |
| |
| img.save('img_'+sys.argv[2]+'_'+sys.argv[1]+'.png') | |
| |
| |
| |
| more ~/mitosearch/Mitosearch/data/fish/lat-long-date.txt |awk -F'\t' '$2~"^[0-9]"{split($2,arr," "); if(arr[2]=="N"){k=arr[1]}else{k=-arr[1]}; if(arr[4]=="E"){k2=arr[3]}else{k2=-arr[3]}; if(flag[k":"k2]==0){flag[k":"k2]=1; print $1"\t"k"\t"k2}}'|awk '{print "echo "$1":"$2" "$3"; docker run -v $PWD:$PWD -w $PWD -i --rm c2997108/python:3.10-staticmap_2 mapwater.py "$3" "$2}'|xargs -I{} bash -c "{}"|paste - - > mapwater.result | |
| |
| cat ~/mitosearch/Mitosearch/data/fish/lat-long-date.txt |awk -F'\t' '$2~"^[0-9]"{split($2,arr," "); if(arr[2]=="N"){k=arr[1]}else{k=-arr[1]}; if(arr[4]=="E"){k2=arr[3]}else{k2=-arr[3]}; print $1"\t"k" "k2}'|more|awk -F'\t' 'FILENAME==ARGV[1]{data[$1]=$2} FILENAME==ARGV[2]{print $1"\t"data[$2]}' <(awk -F'\t' '{split($1,arr,":"); split($2,arr2," "); print arr[2]"\t"arr2[1]}' mapwater.result) /dev/stdin > mapwater.result.txt | |
| |
| #差分のみ取得するバージョン | |
| more ~/mitosearch/Mitosearch/data/fish/lat-long-date.txt |awk -F'\t' '$2~"^[0-9]"{split($2,arr," "); if(arr[2]=="N"){k=arr[1]}else{k=-arr[1]}; if(arr[4]=="E"){k2=arr[3]}else{k2=-arr[3]}; if(flag[k":"k2]==0){flag[k":"k2]=1; print $1"\t"k"\t"k2}}'|awk -F'\t' 'FILENAME==ARGV[1]{a[$1]=1} FILENAME==ARGV[2]&&a[$2" "$3]==0{print $0}' <(awk -F'\t' '{split($1,arr,":"); split($2,arr2," "); print arr[2]"\t"arr2[1]}' mapwater.result) /dev/stdin |awk '{print "echo "$1":"$2" "$3"; docker run -v $PWD:$PWD -w $PWD -i --rm c2997108/python:3.10-staticmap_2 mapwater.py "$3" "$2}'|xargs -I{} bash -c "{}"|paste - - > mapwater.result.new | |
| |
| cat ~/mitosearch/Mitosearch/data/fish/lat-long-date.txt |awk -F'\t' '$2~"^[0-9]"{split($2,arr," "); if(arr[2]=="N"){k=arr[1]}else{k=-arr[1]}; if(arr[4]=="E"){k2=arr[3]}else{k2=-arr[3]}; print $1"\t"k" "k2}'|more|awk -F'\t' 'FILENAME==ARGV[1]{data[$1]=$2} FILENAME==ARGV[2]{print $1"\t"data[$2]}' <(awk -F'\t' '{split($1,arr,":"); split($2,arr2," "); print arr[2]"\t"arr2[1]}' mapwater.result mapwater.result.new) /dev/stdin > mapwater.result.txt | |
| |
| ``` | |
| |
| ## ANEMOMNEデータベース | |
| |
| ``` | |
| wget --http-user=username --http-passwd=pass -r -np --wait=1 https://db.anemone.bio/dist/MiFish/ --no-check-certificate | |
| find ../db.anemone.bio/|grep fastq.xz|while read i; do ln -s $i `echo $i|sed 's%../db.anemone.bio/dist/MiFish/ANEMONE/%%; s%/%___%g'`; done | |
| find ../db.anemone.bio/|grep sample.tsv.xz|while read i; do ln -s $i `echo $i|sed 's%../db.anemone.bio/dist/MiFish/ANEMONE/%%; s%/%___%g'`; done | |
| for i in *; do xzcat $i > `echo $i|sed 's/.xz$//'`; done | |
| |
| ``` | |