Skip to content

use smart_open for doc2vec-IMDB #1278

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tmylk merged 16 commits into from
May 2, 2017
Merged

use smart_open for doc2vec-IMDB #1278

Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
1aa3f33
fix the compatibility between python2 & 3
robotcator Mar 17, 2017
24e6331
Merge https://github.com/RaRe-Technologies/gensim into fix-word2vec-n…
robotcator Mar 18, 2017
f6f571f
require explicit corpus size, epochs for train()
gojomo Feb 9, 2017
5e9529b
make all train() calls use explicit count, epochs
gojomo Feb 9, 2017
5c24a90
add tests to make sure that ValueError is indeed thrown
robotcator Mar 23, 2017
c89f285
update test
robotcator Mar 24, 2017
10ff8a5
fix the word2vec's reset_from()
robotcator Mar 25, 2017
a6312ca
Merge branch 'fix-word2vec' into fix-word2vec-notebook
robotcator Mar 29, 2017
be5216a
Merge branch 'develop' of https://github.com/RaRe-Technologies/gensim…
robotcator Mar 29, 2017
504bd09
require explicit corpus size, epochs for train()
gojomo Feb 9, 2017
43f9689
make all train() calls use explicit count, epochs
gojomo Feb 9, 2017
49e3d00
update notebooks
robotcator Mar 29, 2017
c9eab32
fix some error
robotcator Mar 29, 2017
8024eb5
fix test error
robotcator Mar 29, 2017
567af2b
Merge branch 'develop' of https://github.com/RaRe-Technologies/gensim…
robotcator Apr 9, 2017
dc51aa3
drop codecs, use smart_open
robotcator Apr 12, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 25 additions & 15 deletions docs/notebooks/doc2vec-IMDB.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,13 +50,21 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 1,
"metadata": {
"collapsed": true,
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"total running time: 0.0014970000000000816\n"
]
}
],
"source": [
"import locale\n",
"import glob\n",
Expand All @@ -65,6 +73,7 @@
"import tarfile\n",
"import sys\n",
"import codecs\n",
"import smart_open\n",
"\n",
"dirname = 'aclImdb'\n",
"filename = 'aclImdb_v1.tar.gz'\n",
Expand Down Expand Up @@ -113,11 +122,11 @@
" output = fol.replace('/', '-') + '.txt'\n",
"\n",
" # Is there a better pattern to use?\n",
" txt_files = glob.glob('/'.join([dirname, fol, '*.txt']))\n",
" txt_files = glob.glob(os.path.join(dirname, fol, '*.txt'))\n",
"\n",
" for txt in txt_files:\n",
" with codecs.open(txt, 'r', encoding='utf-8') as t:\n",
" t_clean = t.read()\n",
" with smart_open.smart_open(txt, \"rb\") as t:\n",
" t_clean = t.read().decode(\"utf-8\")\n",
"\n",
" for c in control_chars:\n",
" t_clean = t_clean.replace(c, ' ')\n",
Expand All @@ -127,15 +136,16 @@
" temp += \"\\n\"\n",
"\n",
" temp_norm = normalize_text(temp)\n",
" with codecs.open('/'.join([dirname, output]), 'w', encoding='utf-8') as n:\n",
" n.write(temp_norm)\n",
"\n",
" with smart_open.smart_open(os.path.join(dirname, output), \"wb\") as n:\n",
" n.write(temp_norm.encode(\"utf-8\"))\n",
"\n",
" alldata += temp_norm\n",
"\n",
" with codecs.open('/'.join([dirname, 'alldata-id.txt']), 'w', encoding='utf-8') as f:\n",
" with smart_open.smart_open(os.path.join(dirname, 'alldata-id.txt'), 'wb') as f:\n",
" for idx, line in enumerate(alldata.splitlines()):\n",
" num_line = u\"_*{0} {1}\\n\".format(idx, line)\n",
" f.write(num_line)\n",
" f.write(num_line.encode(\"utf-8\"))\n",
"\n",
"end = time.clock()\n",
"print (\"total running time: \", end-start)"
Expand Down Expand Up @@ -1092,21 +1102,21 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"display_name": "Python 3",
"language": "python",
"name": "python2"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
"pygments_lexer": "ipython3",
"version": "3.4.3"
}
},
"nbformat": 4,
Expand Down