和名ファイルの準備

20210718_JAFList.xlsxをダウンロードしておく。「和名」と「学名」をサクラエディタに張り付けて、「“」の文字をCtrl＋Rで「」に置換して削除してから（セル内改行があると、改行コードLFで改行された”“で囲まれた文字が出現するための対応。WinはCR+LFがデフォルト）、改行コードをLFで統一して保存する。

m50v251n3の/home/yoshitake/mitosearch/Mitosearch/data/fish/japname.txtに転送する。

#本当はgrep -v "^和名なし"を入れたいけど、不一致が出るので省略
#cat japname.txt |grep -v "^和名なし"|grep -v "^[a-zA-Z]"|awk -F'\t' '{split($2,arr," "); print arr[1]" "arr[2]"\t"$0}' > japname2.txt
tac japname.txt |grep -v "^[a-zA-Z]"|awk -F'\t' '{split($2,arr," "); print arr[1]" "arr[2]"\t"$0}' > japname2.txt


blastdbcmd -db ~/mitosearch_related_files/db/complete_partial_mitogenomes.fa -entry all|grep ">"|sed 's/ [(].*//'|awk '{split($0,arr,"|"); print arr[2]"\t"arr[3]}'|awk -F'\t' 'FILENAME==ARGV[1]{jap[$1]=$2} FILENAME==ARGV[2]{split($2,arr," "); print jap[arr[1]" "arr[2]]"\t"$0}' japname2.txt /dev/stdin > japname3.txt
#cat /home/yoshitake/mitosearch_update_db/db/complete_partial_mitogenomes.fa|grep ">"|sed 's/ [(].*//'|awk '{split($0,arr,"|"); print arr[2]"\t"arr[3]}'|awk -F'\t' 'FILENAME==ARGV[1]{jap[$1]=$2} FILENAME==ARGV[2]{split($2,arr," "); print jap[arr[1]" "arr[2]]"\t"$0}' japname2.txt /dev/stdin > japname3.txt

zcat ~/yoshitake/nucl_gb.accession2taxid.gz |awk -F'\t' 'FILENAME==ARGV[1]{a[$2]=1} FILENAME==ARGV[2]{if(a[$1]==1){b[$1]=$3}else{delete a[$1]}} FILENAME==ARGV[3]{name[$1]=$2} FILENAME==ARGV[4]{print $0"\t"name[b[$2]]}' japname3.txt /dev/stdin ~/yoshitake/names.dmp.sname.path ./japname3.txt > japname4.txt

more japname4.txt|awk -F'\t' '$4!=""'|sed 's/.*\troot;//'|cut -f 10- -d";"|sort|uniq -c|sed 's/^ *//; s/ /\t/' > japname4.txt2

j=12; more japname4.txt2|awk -F'\t' -v j=$j '{split($2,arr,";"); str=arr[1]; for(i=2;i<j;i++){str=str";"arr[i]}; cnt[str]+=$1; all+=$1} END{for(i in cnt){print cnt[i]"\t"i"\t"all}}'|sort -k1,1nr|more
#j=11, 12,,,などとやってトップが1/8以上になるところを調べる 688858/8=86107
#機械的にクラスター設定をしようと思ったけど、結局伊藤くんが設定してくれていた分類をつかうことにした。

more japname4.txt|awk -F'\t' '{a[$3]++; if(a[$3]==1){print $0}}'|awk -F'\t' 'BEGIN{str="Vertebrata;Gnathostomata;Chondrichthyes,Vertebrata;Gnathostomata;Teleostomi,Vertebrata;Gnathostomata;Teleostomi;Euteleostomi;Actinopterygii;Actinopteri;Neopterygii;Teleostei;Osteoglossocephalai,Vertebrata;Gnathostomata;Teleostomi;Euteleostomi;Actinopterygii;Actinopteri;Neopterygii;Teleostei;Elopocephalai,Vertebrata;Gnathostomata;Teleostomi;Euteleostomi;Actinopterygii;Actinopteri;Neopterygii;Teleostei;Osteoglossocephalai;Clupeocephala;Euteleosteomorpha,Vertebrata;Gnathostomata;Teleostomi;Euteleostomi;Actinopterygii;Actinopteri;Neopterygii;Teleostei;Osteoglossocephalai;Clupeocephala;Euteleosteomorpha;Neoteleostei,Vertebrata;Gnathostomata;Teleostomi;Euteleostomi;Actinopterygii;Actinopteri;Neopterygii;Teleostei;Osteoglossocephalai;Clupeocephala;Euteleosteomorpha;Neoteleostei,Vertebrata;Gnathostomata;Teleostomi;Euteleostomi;Actinopterygii;Actinopteri;Neopterygii;Teleostei;Osteoglossocephalai;Clupeocephala;Euteleosteomorpha;Neoteleostei;Eurypterygia;Ctenosquamata;Acanthomorphata;Euacanthomorphacea;Percomorphaceae;Eupercaria"; split(str,arr,",")} {col=0; for(i=1;i<=8;i++){if($4~arr[i]){col=i}}; print col"\t"$0}'|awk -F'\t' '{if($2!=""){print $2":"$4" \t"$1}else{print $4" \t"$1}}'|sort|uniq > japname5.txt

cp japname5.txt classifylist.txt

MitoFishDBのすべての魚のtaxonomy pathを取得

blastdbcmd -db ~/mitosearch_related_files/db/complete_partial_mitogenomes.fa -entry all|grep ">"|cut -f 2 -d '|'|awk -F'\t' 'FILENAME==ARGV[1]{a[$1]=1} FILENAME==ARGV[2]&&$1 in a{b[$1]=$3} FILENAME==ARGV[3]{c[$1]=$2} FILENAME==ARGV[4]{print $1"\t"c[b[$1]]}' /dev/stdin <(zcat ~/yoshitake/nucl_gb.accession2taxid.gz ) ~/yoshitake/names.dmp.sname.path <(blastdbcmd -db ~/mitosearch_related_files/db/complete_partial_mitogenomes.fa -entry all|grep ">"|cut -f 2 -d '|') > ~/yoshitake/complete_partial_mitogenomes.fa.taxpath
more ~/yoshitake/complete_partial_mitogenomes.fa.taxpath|cut -f 2|sort|uniq > ~/yoshitake/complete_partial_mitogenomes.fa.taxpath2

cd /home/yoshitake/mitosearch_related_files/script
./grouping.js ~/yoshitake/complete_partial_mitogenomes.fa.taxpath2

more japname3.txt |awk -F'\t' '{if($1!=""){print $1":"$3"\t"$2}else{print $3"\t"$2}}'| awk -F'\t' 'a[$1]==""{a[$1]=1; print $0}' > japname3.txt2
awk -F'\t' 'FILENAME==ARGV[1]{tax2col[$1]=$2} FILENAME==ARGV[2]{mitoid2tax[$1]=$2} FILENAME==ARGV[3]{print $1" \t"tax2col[mitoid2tax[$2]]}' classifylist.txt complete_partial_mitogenomes.fa.taxpath japname3.txt2 > japname3.txt2.class
cp japname3.txt2.class ~/mitosearch_dev/mitosearch/Mitosearch/data/fish/classifylist.txt

和名を付与しなおし

more ~/mitosearch/Mitosearch/data/fish/classifylist.txt|sed 's/^[^:]*://'|awk -F'\t' '{print $2"\t"$1}'|awk -F"\t" '{split($2,arr,";");if(arr[length(arr)]~" x "){split(arr[length(arr)],array," x ");print $1"\t"array[length(array)];}else{print $1"\t"arr[length(arr)];}}' > tmp

awk -F"\t" '
 {if(FILENAME==ARGV[1]){list[$2]=$1;}if(FILENAME==ARGV[2]){for(i in list){if($2~i){list2[$1":"i]=list[i];delete list[i];}}}}
 END{PROCINFO["sorted_in"]="@val_num_desc";for(i in list2){print list2[i]"\t"i;}}
' tmp ../db/scientificname2japanesename_complete.csv > tmp2

awk -F"\t" '
 {if(FILENAME==ARGV[1]){list[$2]=$1;}if(FILENAME==ARGV[2]){for(i in list){if($2~i){delete list[i];}}}}
 END{for(i in list){print list[i]"\t"i;}}
' tmp tmp2 > tmp3
awk -F"\t" '
 {if(FILENAME==ARGV[1]){list[$2]=$1;}if(FILENAME==ARGV[2]){for(i in list){split(i,array," ");if($2~array[1]){list[$3":"i]=list[i];delete list[i];}}}}
 END{PROCINFO["sorted_in"]="@val_num_desc";for(i in list){print list[i]"\t"i;}}
' tmp3 ../db/scientificname2japanesename_complete.csv > tmp4
cat tmp2 tmp4|awk -F"\t" '{list[$2]=$1;}END{PROCINFO["sorted_in"]="@val_num_desc";for(i in list){print list[i]"\t"i;}}'|awk -F'\t' '{print $2"\t"$1}'|sed 's/ \t/\t/' > tmp.cla

mv tmp.cla ~/mitosearch/Mitosearch/data/fish/classifylist.txt

more ~/mitosearch/Mitosearch/data/fish/classifylist.txt|~/tool/grep-3.8/bin/grep "^[ア-ン]"|cut -f 2- -d:|cut -f 1 > /tmp/a
more ~/mitosearch/Mitosearch/data/fish/classifylist.txt|~/tool/grep-3.8/bin/grep -v "^[ア-ン]"|cut -f 1 > /tmp/b
cat /tmp/a /tmp/b |sort|awk 'NF>1' > /tmp/spname.txt
#漢字の和名が残っていたので手動で削除

for i in *.input; do awk -F: '{if($1~"[あ-ン]"){str=$2; for(i=3;i<=NF;i++){str=str":"$i}; print str}else{print $0}}' $i > /tmp/mitoinput/$i; done

(cat ~/mitosearch/Mitosearch/data/fish/classifylist.txt|~/tool/grep-3.8/bin/grep ”[ア-ンあ-ん]“|cut -f 2- -d:; cat ~/mitosearch/Mitosearch/data/fish/classifylist.txt|~/tool/grep-3.8/bin/grep -v ”[ア-ンあ-ん]“) > /tmp/classifylist_plain.txt ```