差分

このページの2つのバージョン間の差分を表示します。

--- mitosearch色分け情報の作成 [2023/02/08 07:50] – [英語のcommon name付与] 133.11.50.163
+++ mitosearch色分け情報の作成 [Unknown date] (現在) – 削除 - 外部編集 (Unknown date) 127.0.0.1
@@ 行 1: / 行 1: @@
-# 和名ファイルの準備
-`20210718_JAFList.xlsx`をダウンロードしておく。「和名」と「学名」をサクラエディタに張り付けて、「"」の文字をCtrl＋Rで「」に置換して削除してから（セル内改行があると、改行コードLFで改行された""で囲まれた文字が出現するための対応。WinはCR+LFがデフォルト）、改行コードをLFで統一して保存する。
-m50v251n3の`/home/yoshitake/mitosearch/Mitosearch/data/fish/japname.txt`に転送する。
-```
-#本当はgrep -v "^和名なし"を入れたいけど、不一致が出るので省略
-#cat japname.txt |grep -v "^和名なし"|grep -v "^[a-zA-Z]"|awk -F'\t' '{split($2,arr," "); print arr[1]" "arr[2]"\t"$0}' > japname2.txt
-tac japname.txt |grep -v "^[a-zA-Z]"|awk -F'\t' '{split($2,arr," "); print arr[1]" "arr[2]"\t"$0}' > japname2.txt
-blastdbcmd -db ~/mitosearch_related_files/db/complete_partial_mitogenomes.fa -entry all|grep ">"|sed 's/ [(].*//'|awk '{split($0,arr,"|"); print arr[2]"\t"arr[3]}'|awk -F'\t' 'FILENAME==ARGV[1]{jap[$1]=$2} FILENAME==ARGV[2]{split($2,arr," "); print jap[arr[1]" "arr[2]]"\t"$0}' japname2.txt /dev/stdin > japname3.txt
-#cat /home/yoshitake/mitosearch_update_db/db/complete_partial_mitogenomes.fa|grep ">"|sed 's/ [(].*//'|awk '{split($0,arr,"|"); print arr[2]"\t"arr[3]}'|awk -F'\t' 'FILENAME==ARGV[1]{jap[$1]=$2} FILENAME==ARGV[2]{split($2,arr," "); print jap[arr[1]" "arr[2]]"\t"$0}' japname2.txt /dev/stdin > japname3.txt
-zcat ~/yoshitake/nucl_gb.accession2taxid.gz |awk -F'\t' 'FILENAME==ARGV[1]{a[$2]=1} FILENAME==ARGV[2]{if(a[$1]==1){b[$1]=$3}else{delete a[$1]}} FILENAME==ARGV[3]{name[$1]=$2} FILENAME==ARGV[4]{print $0"\t"name[b[$2]]}' japname3.txt /dev/stdin ~/yoshitake/names.dmp.sname.path ./japname3.txt > japname4.txt
-more japname4.txt|awk -F'\t' '$4!=""'|sed 's/.*\troot;//'|cut -f 10- -d";"|sort|uniq -c|sed 's/^ *//; s/ /\t/' > japname4.txt2
-j=12; more japname4.txt2|awk -F'\t' -v j=$j '{split($2,arr,";"); str=arr[1]; for(i=2;i<j;i++){str=str";"arr[i]}; cnt[str]+=$1; all+=$1} END{for(i in cnt){print cnt[i]"\t"i"\t"all}}'|sort -k1,1nr|more
-#j=11, 12,,,などとやってトップが1/8以上になるところを調べる 688858/8=86107
-#機械的にクラスター設定をしようと思ったけど、結局伊藤くんが設定してくれていた分類をつかうことにした。
-more japname4.txt|awk -F'\t' '{a[$3]++; if(a[$3]==1){print $0}}'|awk -F'\t' 'BEGIN{str="Vertebrata;Gnathostomata;Chondrichthyes,Vertebrata;Gnathostomata;Teleostomi,Vertebrata;Gnathostomata;Teleostomi;Euteleostomi;Actinopterygii;Actinopteri;Neopterygii;Teleostei;Osteoglossocephalai,Vertebrata;Gnathostomata;Teleostomi;Euteleostomi;Actinopterygii;Actinopteri;Neopterygii;Teleostei;Elopocephalai,Vertebrata;Gnathostomata;Teleostomi;Euteleostomi;Actinopterygii;Actinopteri;Neopterygii;Teleostei;Osteoglossocephalai;Clupeocephala;Euteleosteomorpha,Vertebrata;Gnathostomata;Teleostomi;Euteleostomi;Actinopterygii;Actinopteri;Neopterygii;Teleostei;Osteoglossocephalai;Clupeocephala;Euteleosteomorpha;Neoteleostei,Vertebrata;Gnathostomata;Teleostomi;Euteleostomi;Actinopterygii;Actinopteri;Neopterygii;Teleostei;Osteoglossocephalai;Clupeocephala;Euteleosteomorpha;Neoteleostei,Vertebrata;Gnathostomata;Teleostomi;Euteleostomi;Actinopterygii;Actinopteri;Neopterygii;Teleostei;Osteoglossocephalai;Clupeocephala;Euteleosteomorpha;Neoteleostei;Eurypterygia;Ctenosquamata;Acanthomorphata;Euacanthomorphacea;Percomorphaceae;Eupercaria"; split(str,arr,",")} {col=0; for(i=1;i<=8;i++){if($4~arr[i]){col=i}}; print col"\t"$0}'|awk -F'\t' '{if($2!=""){print $2":"$4" \t"$1}else{print $4" \t"$1}}'|sort|uniq > japname5.txt
-cp japname5.txt classifylist.txt
-```
-# 2022-09
-MitoFishDBのすべての魚のtaxonomy pathを取得
-```
-blastdbcmd -db ~/mitosearch_related_files/db/complete_partial_mitogenomes.fa -entry all|grep ">"|cut -f 2 -d '|'|awk -F'\t' 'FILENAME==ARGV[1]{a[$1]=1} FILENAME==ARGV[2]&&$1 in a{b[$1]=$3} FILENAME==ARGV[3]{c[$1]=$2} FILENAME==ARGV[4]{print $1"\t"c[b[$1]]}' /dev/stdin <(zcat ~/yoshitake/nucl_gb.accession2taxid.gz ) ~/yoshitake/names.dmp.sname.path <(blastdbcmd -db ~/mitosearch_related_files/db/complete_partial_mitogenomes.fa -entry all|grep ">"|cut -f 2 -d '|') > ~/yoshitake/complete_partial_mitogenomes.fa.taxpath
-more ~/yoshitake/complete_partial_mitogenomes.fa.taxpath|cut -f 2|sort|uniq > ~/yoshitake/complete_partial_mitogenomes.fa.taxpath2
-cd /home/yoshitake/mitosearch_related_files/script
-./grouping.js ~/yoshitake/complete_partial_mitogenomes.fa.taxpath2
-```
-```
-more japname3.txt |awk -F'\t' '{if($1!=""){print $1":"$3"\t"$2}else{print $3"\t"$2}}'| awk -F'\t' 'a[$1]==""{a[$1]=1; print $0}' > japname3.txt2
-awk -F'\t' 'FILENAME==ARGV[1]{tax2col[$1]=$2} FILENAME==ARGV[2]{mitoid2tax[$1]=$2} FILENAME==ARGV[3]{print $1" \t"tax2col[mitoid2tax[$2]]}' classifylist.txt complete_partial_mitogenomes.fa.taxpath japname3.txt2 > japname3.txt2.class
-cp japname3.txt2.class ~/mitosearch_dev/mitosearch/Mitosearch/data/fish/classifylist.txt
-```
-# 2023-01
-和名を付与しなおし
-```
-more ~/mitosearch/Mitosearch/data/fish/classifylist.txt|sed 's/^[^:]*://'|awk -F'\t' '{print $2"\t"$1}'|awk -F"\t" '{split($2,arr,";");if(arr[length(arr)]~" x "){split(arr[length(arr)],array," x ");print $1"\t"array[length(array)];}else{print $1"\t"arr[length(arr)];}}' > tmp
-awk -F"\t" '
- {if(FILENAME==ARGV[1]){list[$2]=$1;}if(FILENAME==ARGV[2]){for(i in list){if($2~i){list2[$1":"i]=list[i];delete list[i];}}}}
- END{PROCINFO["sorted_in"]="@val_num_desc";for(i in list2){print list2[i]"\t"i;}}
-' tmp ../db/scientificname2japanesename_complete.csv > tmp2
-awk -F"\t" '
- {if(FILENAME==ARGV[1]){list[$2]=$1;}if(FILENAME==ARGV[2]){for(i in list){if($2~i){delete list[i];}}}}
- END{for(i in list){print list[i]"\t"i;}}
-' tmp tmp2 > tmp3
-awk -F"\t" '
- {if(FILENAME==ARGV[1]){list[$2]=$1;}if(FILENAME==ARGV[2]){for(i in list){split(i,array," ");if($2~array[1]){list[$3":"i]=list[i];delete list[i];}}}}
- END{PROCINFO["sorted_in"]="@val_num_desc";for(i in list){print list[i]"\t"i;}}
-' tmp3 ../db/scientificname2japanesename_complete.csv > tmp4
-cat tmp2 tmp4|awk -F"\t" '{list[$2]=$1;}END{PROCINFO["sorted_in"]="@val_num_desc";for(i in list){print list[i]"\t"i;}}'|awk -F'\t' '{print $2"\t"$1}'|sed 's/ \t/\t/' > tmp.cla
-mv tmp.cla ~/mitosearch/Mitosearch/data/fish/classifylist.txt
-```
-# Common name付与
-## 魚種リストを作成
-```
-more ~/mitosearch/Mitosearch/data/fish/classifylist.txt|~/tool/grep-3.8/bin/grep "^[ア-ン]"|cut -f 2- -d:|cut -f 1 > /tmp/a
-more ~/mitosearch/Mitosearch/data/fish/classifylist.txt|~/tool/grep-3.8/bin/grep -v "^[ア-ン]"|cut -f 1 > /tmp/b
-cat /tmp/a /tmp/b |sort|awk 'NF>1' > /tmp/spname.txt
-#漢字の和名が残っていたので手動で削除
-```
-```
-for i in *.input; do awk -F: '{if($1~"[あ-ン]"){str=$2; for(i=3;i<=NF;i++){str=str":"$i}; print str}else{print $0}}' $i > /tmp/mitoinput/$i; done
-```
-## classifylist.txtの和名除去
-```
-(cat ~/mitosearch/Mitosearch/data/fish/classifylist.txt|~/tool/grep-3.8/bin/grep "[ア-ンあ-ん]"|cut -f 2- -d:;
- cat ~/mitosearch/Mitosearch/data/fish/classifylist.txt|~/tool/grep-3.8/bin/grep -v "[ア-ンあ-ん]") > /tmp/classifylist_plain.txt
-```
-## 英語のcommon name付与
-- genbank common name > common name
-- 名前が長い方を優先
-でつける
-```
-more names.dmp|awk -F'\t' '{OFS="\t"; $2=length($3); print $0}'|grep "common name"|sort -t$'\t' -k1,1n -k7,7 -k2,2n > names.dmp.common_name
-awk -F'\t' 'FILENAME==ARGV[1]{name[$1]=$3} FILENAME==ARGV[2]{print $1"\t"$3"\t"name[$1]}' names.dmp.common_name names.dmp.sname > names.dmp.sname-common_name
-```