From b5825f2d3d5346e3da0a805f061ec6604bbecbad Mon Sep 17 00:00:00 2001 From: "Guan-Ming (Wesley) Chiu" <105915352+guan404ming@users.noreply.github.com> Date: Thu, 30 Oct 2025 15:27:50 +0800 Subject: [PATCH 1/2] Integrate pre-commit --- .github/workflows/pre-commit.yml | 42 ++++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 11 +++++++++ pyproject.toml | 2 +- 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/pre-commit.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000000..e8240a4d91 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Pre-commit + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Poetry and dependencies + run: | + pip install poetry + poetry install --extras dev + + - name: Run pre-commit hooks + run: poetry run pre-commit run --all-files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..f76ebdbbbd --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,11 @@ +repos: + # Check yaml, end of file, trailing whitespace + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-yaml + name: check yaml + - id: end-of-file-fixer + name: fix end of file + - id: trailing-whitespace + name: trim trailing whitespace diff --git a/pyproject.toml b/pyproject.toml index ae509a7ec5..36ae5ee4b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=8.1.1", "ruff>=0.13.1"] +dev = ["pytest>=8.1.1", "ruff>=0.13.1", "pre-commit>=3.0.0"] [tool.pytest.ini_options] testpaths = ["testing"] From b09fb98ab192382ce0dd146677fd95727a0715ff Mon Sep 17 00:00:00 2001 From: "Guan-Ming (Wesley) Chiu" <105915352+guan404ming@users.noreply.github.com> Date: Tue, 4 Nov 2025 21:40:39 +0800 Subject: [PATCH 2/2] Integrate pre-commit --- .github/ISSUE_TEMPLATE/bug_report.md | 14 +- .github/ISSUE_TEMPLATE/documentation_issue.md | 8 +- .github/ISSUE_TEMPLATE/feature_request.md | 10 +- .github/ISSUE_TEMPLATE/issue_template.md | 64 +- .github/PULL_REQUEST_TEMPLATE | 11 +- .github/scripts/checks/general_review.py | 36 +- .github/workflows/notebook-testing.yml | 8 +- .github/workflows/pre-commit.yml | 5 +- .github/workflows/website.yml | 5 +- .gitignore | 2 +- doap_Mahout.rdf | 8 +- docs/api.md | 83 ++- docs/getting_started_with_qumat.md | 4 +- docs/index.md | 16 +- docs/p_q_c.md | 566 +++++++++--------- docs/qumat_gap_analysis_for_pqc.md | 312 +++++----- .../Classification/bankmarketing-example.md | 4 +- .../Classification/bayesian-commandline.md | 5 +- .../Classification/bayesian.md | 126 ++-- .../Classification/breiman-example.md | 7 +- .../Classification/class-discovery.md | 14 +- .../Classification/hidden-markov-models.md | 28 +- .../Classification/logistic-regression.md | 11 +- .../Map_Reduce_Folder/Classification/mlp.md | 39 +- .../Classification/partial-implementation.md | 6 +- .../restricted-boltzmann-machines.md | 8 +- .../Classification/twenty-newsgroups.md | 81 ++- .../wikipedia-classifier-example.md | 5 +- .../about/distributed-matrix-math.md | 1 - website/Map_Reduce_Folder/algorithms/d-als.md | 11 +- website/Map_Reduce_Folder/algorithms/d-qr.md | 11 +- .../Map_Reduce_Folder/algorithms/d-spca.md | 6 +- .../Map_Reduce_Folder/algorithms/d-ssvd.md | 37 +- .../algorithms/intro-cooccurrence-spark.md | 158 ++--- .../algorithms/recommender-overview.md | 2 +- .../algorithms/spark-naive-bayes.md | 105 ++-- .../Map_Reduce_Folder/basics/algorithms.md | 28 +- .../Map_Reduce_Folder/basics/collocations.md | 121 ++-- .../basics/creating-vectors-from-text.md | 297 +++++---- .../basics/mahout-collections.md | 8 +- .../basics/matrix-and-vector-needs.md | 2 +- .../clustering/canopy-clustering.md | 7 +- .../clustering/canopy-commandline.md | 33 +- .../clustering/cluster-dumper.md | 38 +- .../clustering-of-synthetic-control-data.md | 5 +- .../clustering/clusteringyourdata.md | 39 +- .../clustering/expectation-maximization.md | 1 - .../clustering/fuzzy-k-means-commandline.md | 88 ++- .../clustering/fuzzy-k-means.md | 10 +- .../clustering/hierarchical-clustering.md | 3 +- .../clustering/k-means-commandline.md | 69 ++- .../clustering/latent-dirichlet-allocation.md | 8 +- .../clustering/lda-commandline.md | 55 +- .../clustering/llr---log-likelihood-ratio.md | 8 +- .../clustering/streaming-k-means.md | 172 +++--- .../clustering/viewing-results.md | 2 +- .../clustering/visualizing-sample-clusters.md | 2 +- .../developers/buildingmahout.md | 41 +- .../developers/developer-resources.md | 2 +- .../Map_Reduce_Folder/developers/github.md | 88 +-- .../Map_Reduce_Folder/developers/githubPRs.md | 40 +- website/Map_Reduce_Folder/developers/gsoc.md | 6 +- .../developers/how-to-become-a-committer.md | 2 +- .../developers/how-to-release.md | 10 +- .../developers/how-to-update-the-website.md | 24 +- .../developers/issue-tracker.md | 2 +- .../developers/patch-check-list.md | 8 +- .../developers/thirdparty-dependencies.md | 2 +- .../developers/version-control.md | 6 +- .../dim-reduction/dimensional-reduction.md | 66 +- .../Map_Reduce_Folder/dim-reduction/ssvd.md | 33 +- .../algorithms/clustering/canopy/index.md | 8 +- .../algorithms/clustering/distance-metrics.md | 28 +- .../latest/algorithms/linear-algebra/d-qr.md | 27 +- .../algorithms/linear-algebra/d-spca.md | 8 +- .../algorithms/linear-algebra/d-ssvd.md | 39 +- .../latest/algorithms/linear-algebra/index.md | 3 +- .../map-reduce/classification/bayesian.md | 128 ++-- .../classification/class-discovery.md | 14 +- .../map-reduce/classification/collocations.md | 121 ++-- .../classification/hidden-markov-models.md | 28 +- .../classification/logistic-regression.md | 11 +- .../classification/mahout-collections.md | 8 +- .../map-reduce/classification/mlp.md | 39 +- .../classification/partial-implementation.md | 6 +- .../restricted-boltzmann-machines.md | 8 +- .../clustering/canopy-clustering.md | 7 +- .../map-reduce/clustering/cluster-dumper.md | 38 +- .../clustering/expectation-maximization.md | 1 - .../map-reduce/clustering/fuzzy-k-means.md | 10 +- .../clustering/hierarchical-clustering.md | 3 +- .../clustering/latent-dirichlet-allocation.md | 8 +- .../clustering/llr---log-likelihood-ratio.md | 7 +- .../clustering/streaming-k-means.md | 172 +++--- .../algorithms/preprocessors/AsFactor.md | 7 +- .../algorithms/preprocessors/MeanCenter.md | 5 +- .../preprocessors/StandardScaler.md | 8 +- .../latest/algorithms/preprocessors/index.md | 1 - .../latest/algorithms/recommenders/cco.md | 158 ++--- .../latest/algorithms/recommenders/d-als.md | 13 +- .../latest/algorithms/recommenders/index.md | 2 +- .../algorithms/regression/fittness-tests.md | 2 - .../docs/latest/algorithms/regression/ols.md | 6 +- .../latest/algorithms/regression/ridge.md | 2 +- .../serial-correlation/cochrane-orcutt.md | 25 +- .../docs/latest/algorithms/template.md | 5 - .../docs/latest/distributed/flink-bindings.md | 4 +- .../docs/latest/distributed/h2o-internals.md | 2 +- .../latest/distributed/spark-bindings/faq.md | 36 +- .../distributed/spark-bindings/index.md | 31 +- .../Map_Reduce_Folder/docs/latest/index.md | 61 +- .../docs/latest/mahout-samsara/faq.md | 34 +- .../mahout-samsara/in-core-reference.md | 124 ++-- .../mahout-samsara/out-of-core-reference.md | 99 ++- .../docs/latest/quickstart.md | 10 +- .../getting-started-with-zepplin/index.md | 6 +- .../classify-a-doc-from-the-shell.md | 52 +- .../environment/h2o-internals.md | 2 +- .../environment/how-to-build-an-app.md | 50 +- .../environment/in-core-reference.md | 126 ++-- .../environment/out-of-core-reference.md | 101 ++-- .../environment/spark-internals.md | 4 +- .../flinkbindings/flink-internals.md | 6 +- .../playing-with-samsara-flink.md | 30 +- .../general/books-tutorials-and-talks.md | 10 +- website/Map_Reduce_Folder/general/faq.md | 4 +- website/Map_Reduce_Folder/general/glossary.md | 2 +- .../general/mahout-benchmarks.md | 2 - .../Map_Reduce_Folder/general/mahout-wiki.md | 5 +- .../general/powered-by-mahout.md | 4 +- .../general/professional-support.md | 2 +- .../general/reference-reading.md | 13 +- .../general/release-notes.md | 24 +- .../misc/parallel-frequent-pattern-mining.md | 4 +- website/Map_Reduce_Folder/misc/testing.md | 9 +- .../using-mahout-with-python-via-jpype.md | 111 ++-- .../news-and-events/events.md | 3 - .../Map_Reduce_Folder/news-and-events/news.md | 3 +- .../recommender/intro-als-hadoop.md | 21 +- .../recommender/intro-cooccurrence-spark.md | 158 ++--- .../recommender/intro-itembased-hadoop.md | 2 +- .../recommender/matrix-factorization.md | 79 ++- .../recommender/recommender-documentation.md | 28 +- .../recommender-first-timer-faq.md | 4 +- .../recommender/userbased-5-minutes.md | 21 +- .../Map_Reduce_Folder/sparkbindings/faq.md | 34 +- .../Map_Reduce_Folder/sparkbindings/home.md | 32 +- .../sparkbindings/play-with-shell.md | 80 +-- .../tutorials/cco-last-fm/index.md | 38 +- .../tutorials/eigenfaces/index.md | 10 +- .../intro-cooccurrence-spark/index.md | 174 +++--- .../classification/bankmarketing-example.md | 6 +- .../classification/breiman-example.md | 7 +- .../classification/twenty-newsgroups.md | 80 ++- .../wikipedia-classifier-example.md | 5 +- .../clustering/canopy-commandline.md | 33 +- .../clustering-of-synthetic-control-data.md | 5 +- .../clustering/clusteringyourdata.md | 39 +- .../clustering/fuzzy-k-means-commandline.md | 88 ++- .../clustering/k-means-commandline.md | 69 ++- .../map-reduce/clustering/lda-commandline.md | 55 +- .../map-reduce/clustering/viewing-results.md | 2 +- .../clustering/visualizing-sample-clusters.md | 2 +- .../misc/parallel-frequent-pattern-mining.md | 4 +- .../tutorials/map-reduce/misc/testing.md | 9 +- .../using-mahout-with-python-via-jpype.md | 111 ++-- .../recommender/intro-als-hadoop.md | 21 +- .../recommender/intro-cooccurrence-spark.md | 172 +++--- .../recommender/intro-itembased-hadoop.md | 4 +- .../recommender/matrix-factorization.md | 79 ++- .../map-reduce/recommender/quickstart.md | 2 +- .../recommender/recommender-documentation.md | 28 +- .../recommender-first-timer-faq.md | 4 +- .../recommender/userbased-5-minutes.md | 21 +- .../misc/contributing-algos/index.md | 190 +++--- .../tutorials/misc/how-to-build-an-app.md | 48 +- .../misc/mahout-in-zeppelin/index.md | 86 +-- .../samsara/classify-a-doc-from-the-shell.md | 50 +- .../tutorials/samsara/play-with-shell.md | 82 +-- .../playing-with-samsara-flink-batch.md | 30 +- .../tutorials/samsara/spark-naive-bayes.md | 105 ++-- website/_config.yml | 3 +- website/_includes/sidebar.html | 2 +- ...n-Efficient-Quantum-Factoring-Algorithm.md | 42 +- ...ral-Atoms-For-Supervised-Classification.md | 60 +- ...-machine-learning-beyond-kernel-methods.md | 26 +- ...Potential-of-LLMs-for-Quantum-Computing.md | 66 +- website/_posts/2017-12-20-new-website.md | 4 +- website/_posts/2018-06-18-preping-014.md | 15 +- website/_posts/2019-12-14-version-14-1.md | 4 +- .../2020-10-18-version-14-1-released.md | 7 +- .../2020-10-30-weekly-meeting-minutes.md | 8 +- .../2020-11-13-weekly-meeting-minutes.md | 14 +- .../2020-11-20-weekly-meeting-minutes.md | 2 +- .../2020-11-6-weekly-meeting-minutes.md | 2 +- .../2020-12-15-weekly-meeting-minutes.md | 1 - .../2020-12-8-weekly-meeting-minutes.md | 4 +- .../2021-01-05-weekly-meeting-minutes.md | 7 +- .../2021-02-02-weekly-meeting-minutes.md | 3 +- .../_posts/2021-06-01-Zeppelin Quickstart.md | 6 +- website/_posts/2023-02-22-Meeting-Minutes.md | 4 +- website/_posts/2023-03-29-Meeting-Minutes.md | 4 +- website/_posts/2023-07-26-Meeting-Minutes.md | 4 +- website/_posts/2023-08-30-Meeting-Minutes.md | 4 +- website/_posts/2023-09-27-Meeting-Minutes.md | 4 +- website/_posts/2023-10-25-Meeting-Minutes.md | 4 +- website/_posts/2023-11-29-Meeting-Minutes.md | 6 +- website/_posts/2024-01-31-Meeting-Minutes.md | 6 +- website/_posts/2024-02-14-Meeting-Minutes.md | 1 - website/_posts/2024-03-07-Meeting-Minutes.md | 1 - website/_posts/2024-04-24-Meeting-Minutes.md | 4 +- website/_posts/2024-08-30-Meeting-Minutes.md | 1 - website/_posts/2024-09-20-Meeting-Minutes.md | 1 - website/_posts/2024-10-04-Meeting-Minutes.md | 1 - website/_posts/2024-11-01-Meeting-Minutes.md | 1 - website/_posts/2025-08-01-Meeting-Minutes.md | 8 +- website/_posts/2025-10-24-Meeting-Minutes.md | 2 +- website/about/how-to-contribute.md | 34 +- website/assets/asf_new_logo.svg | 2 +- .../vendor/bootstrap/js/bootstrap.min.js | 2 +- website/build_site.sh | 1 - website/community/coc.md | 1 - website/community/mailing-lists.md | 16 +- website/community/who-we-are.md | 34 +- website/docs/README.md | 1 - .../regression/serial-correlation/dw-test.md | 11 +- website/docs/latest/changelog.md | 21 +- .../tutorials/cco-lastfm/cco-lastfm.scala | 4 +- .../misc/contributing-algos/Foo.scala | 2 +- .../users/dim-reduction/ssvd.page/ssvd.R | 36 +- website/documentation/users/index.md | 8 +- website/download/downloads.md | 3 +- website/download/quickstart.md | 10 +- website/index.md | 1 - .../01_introduction/index.md | 54 +- .../03_qubits/index.md | 64 +- .../04_quantum_gates/index.md | 38 +- .../05_quantum_circuits/index.md | 40 +- .../06_quantum_entanglement/index.md | 32 +- .../07_quantum_algorithms/index.md | 90 +-- .../08_quantum_error_correction/index.md | 28 +- .../09_applications/index.md | 200 +++---- .../10_advanced_topics/index.md | 70 +-- website/quantum-computing-primer/index.md | 3 +- 244 files changed, 4060 insertions(+), 4269 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 8972690c7c..d710f054b4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -6,26 +6,26 @@ labels: ["bug"] assignees: [] --- -### Description +### Description A clear and concise description of the bug. -### Steps to Reproduce +### Steps to Reproduce 1. Step 1 2. Step 2 3. Step 3 -### Expected Behavior +### Expected Behavior What should have happened? -### Actual Behavior +### Actual Behavior What actually happened? -### Screenshots or Logs +### Screenshots or Logs Add screenshots or log output if applicable. -### Environment +### Environment - **OS:** e.g., Windows, Linux - **Browser/Tool Version:** e.g., Chrome 96, Python 3.8 -### Additional Context +### Additional Context Any other information that may be helpful. diff --git a/.github/ISSUE_TEMPLATE/documentation_issue.md b/.github/ISSUE_TEMPLATE/documentation_issue.md index 5b1f4bc6cf..4bd0951121 100644 --- a/.github/ISSUE_TEMPLATE/documentation_issue.md +++ b/.github/ISSUE_TEMPLATE/documentation_issue.md @@ -6,14 +6,14 @@ labels: ["docs"] assignees: [] --- -### Description +### Description What is unclear, incorrect, or missing? -### Location of the Issue +### Location of the Issue Provide links or file paths where the issue exists. -### Suggested Improvement +### Suggested Improvement How should this be fixed? -### Additional Context +### Additional Context Anything else that would help improve the documentation. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index dc0edf5958..22a3e75644 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -6,17 +6,17 @@ labels: ["enhancement"] assignees: [] --- -### Summary +### Summary A brief description of the feature. -### Use Case +### Use Case Explain why this feature is useful. -### Proposed Implementation +### Proposed Implementation How do you propose implementing this feature? -### Alternatives Considered +### Alternatives Considered Have you thought of other ways to solve the same problem? -### Additional Context +### Additional Context Any other relevant information or resources. diff --git a/.github/ISSUE_TEMPLATE/issue_template.md b/.github/ISSUE_TEMPLATE/issue_template.md index 16909ae1e8..f82b718313 100644 --- a/.github/ISSUE_TEMPLATE/issue_template.md +++ b/.github/ISSUE_TEMPLATE/issue_template.md @@ -1,32 +1,32 @@ -## Issue Title -Provide a clear and concise title for the issue - -### Description -Describe the issue in detail. Include what the current behavior is and provide any relevant examples or screenshots. - -### Steps to Reproduce -1. First step -2. Second step -3. Third step - -### Expected Behavior -What do you expect to happen? - -### Actual Behavior -What actually happens? Include any error messages or logs if applicable. - -### Environment -- **OS:** e.g. Windows, MacOS, Linux -- **Browser:** e.g. Chrome, Firefox, Safari -- **Version:** e.g. 1.0.0 - -### Suggested Fix -If you have an idea for a fix, please provide details here. - -### Additional Context -Add any other context about the problem here. For example, links to related issues or discussions. - -### Checklist -- [ ] I have searched the existing issues for duplicates. -- [ ] I have provided the information as detailed as possible. -- [ ] I provided relevant screenshots or files if necessary. +## Issue Title +Provide a clear and concise title for the issue + +### Description +Describe the issue in detail. Include what the current behavior is and provide any relevant examples or screenshots. + +### Steps to Reproduce +1. First step +2. Second step +3. Third step + +### Expected Behavior +What do you expect to happen? + +### Actual Behavior +What actually happens? Include any error messages or logs if applicable. + +### Environment +- **OS:** e.g. Windows, MacOS, Linux +- **Browser:** e.g. Chrome, Firefox, Safari +- **Version:** e.g. 1.0.0 + +### Suggested Fix +If you have an idea for a fix, please provide details here. + +### Additional Context +Add any other context about the problem here. For example, links to related issues or discussions. + +### Checklist +- [ ] I have searched the existing issues for duplicates. +- [ ] I have provided the information as detailed as possible. +- [ ] I provided relevant screenshots or files if necessary. diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index c42692d13e..0773600148 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -1,4 +1,4 @@ -### Purpose of PR +### Purpose of PR ### Related Issues or PRs @@ -6,9 +6,9 @@ -### Changes Made +### Changes Made -- [ ] Bug fix +- [ ] Bug fix - [ ] New feature - [ ] Refactoring - [ ] Documentation @@ -16,9 +16,9 @@ - [ ] CI/CD pipeline - [ ] Other -### Breaking Changes +### Breaking Changes -- [ ] Yes +- [ ] Yes - [ ] No ### Checklist @@ -31,4 +31,3 @@ Once all requirements are met, mark as ready for review. --> - [ ] Successfully built and ran all unit tests or manual tests locally - [ ] PR title follows "MAHOUT-XXX: Brief Description" format (if related to an issue) - [ ] Code follows ASF guidelines - diff --git a/.github/scripts/checks/general_review.py b/.github/scripts/checks/general_review.py index c93b1a91c7..8ac0ec8458 100644 --- a/.github/scripts/checks/general_review.py +++ b/.github/scripts/checks/general_review.py @@ -86,24 +86,24 @@ def analyze_code_structure(self, tree): return None def generate_general_impression(self, pr): - prompt = f"""Provide a general impression of this pull request. Consider: -- Overall code quality -- Code organization -- Documentation -- Potential improvements -- Areas that need special attention - -Pull Request Details: -Title: {pr.title} -Description: {pr.body} -Files Changed: {pr.changed_files} -Additions: {pr.additions} -Deletions: {pr.deletions} - -Provide your analysis in markdown format with these sections: -1. Overall Impression -2. Strengths -3. Areas for Improvement + prompt = f"""Provide a general impression of this pull request. Consider: +- Overall code quality +- Code organization +- Documentation +- Potential improvements +- Areas that need special attention + +Pull Request Details: +Title: {pr.title} +Description: {pr.body} +Files Changed: {pr.changed_files} +Additions: {pr.additions} +Deletions: {pr.deletions} + +Provide your analysis in markdown format with these sections: +1. Overall Impression +2. Strengths +3. Areas for Improvement 4. Special Attention Needed""" response = self.client.chat.completions.create( diff --git a/.github/workflows/notebook-testing.yml b/.github/workflows/notebook-testing.yml index 05202fd45b..6cec91bab3 100644 --- a/.github/workflows/notebook-testing.yml +++ b/.github/workflows/notebook-testing.yml @@ -23,13 +23,13 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip pip install nbconvert nbclient ipykernel pip install -e . - name: Run Jupyter Notebooks run: | - for notebook in $(find . -name '*.ipynb'); do - echo "Executing $notebook" - jupyter nbconvert --to notebook --execute --inplace $notebook + for notebook in $(find . -name '*.ipynb'); do + echo "Executing $notebook" + jupyter nbconvert --to notebook --execute --inplace $notebook done diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index e8240a4d91..69dd15a2ca 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -22,8 +22,11 @@ on: branches: [main] jobs: - pre-commit: + test: runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index 05a1b13514..6403e613f3 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -48,10 +48,9 @@ jobs: cp -r $WORKDIR/* . git add . git commit -m "Automatic Site Publish by Buildbot" - git push + git push env: WORKDIR: /tmp/mahout-site - + RUBY_PATH: ~/shared/.rvm GEM_HOME: ${RUBY_PATH}/gems - diff --git a/.gitignore b/.gitignore index 8f11db4cd3..96ff7a75e6 100644 --- a/.gitignore +++ b/.gitignore @@ -48,4 +48,4 @@ __pycache__/ .pytest_cache/ # Poetry -poetry.lock \ No newline at end of file +poetry.lock diff --git a/doap_Mahout.rdf b/doap_Mahout.rdf index 35ffaa5154..bdd72fa6ec 100644 --- a/doap_Mahout.rdf +++ b/doap_Mahout.rdf @@ -1,8 +1,8 @@ To execute ClusterDumper.java, - + * Under mahout-utils, Right-Click on ClusterDumper.java * Choose Run-As, Run Configurations * On the left menu, click on Java Application @@ -80,15 +80,15 @@ Run the clusterdump utility as follows as a standalone Java Program through Ecli In the arguments tab, specify the below arguments - --seqFileDir /examples/output/clusters-10 - --pointsDir /examples/output/clusteredPoints + --seqFileDir /examples/output/clusters-10 + --pointsDir /examples/output/clusteredPoints --output /examples/output/clusteranalyze.txt replace with the actual path of your $MAHOUT_HOME * Hit run to execute the ClusterDumper using Eclipse. Setting breakpoints etc should just work fine. - + Reading the output file - + This will output the clusters into a file called clusteranalyze.txt inside $MAHOUT_HOME/examples/output Sample data will look like diff --git a/website/Map_Reduce_Folder/clustering/clustering-of-synthetic-control-data.md b/website/Map_Reduce_Folder/clustering/clustering-of-synthetic-control-data.md index c2991ed506..e91314da2b 100644 --- a/website/Map_Reduce_Folder/clustering/clustering-of-synthetic-control-data.md +++ b/website/Map_Reduce_Folder/clustering/clustering-of-synthetic-control-data.md @@ -12,13 +12,13 @@ redirect_from: This example will demonstrate clustering of time series data, specifically control charts. [Control charts](http://en.wikipedia.org/wiki/Control_chart) are tools used to determine whether a manufacturing or business process is in a state of statistical control. Such control charts are generated / simulated repeatedly at equal time intervals. A [simulated dataset](http://archive.ics.uci.edu/ml/databases/synthetic_control/synthetic_control.data.html) is available for use in UCI machine learning repository. -A time series of control charts needs to be clustered into their close knit groups. The data set we use is synthetic and is meant to resemble real world information in an anonymized format. It contains six different classes: Normal, Cyclic, Increasing trend, Decreasing trend, Upward shift, Downward shift. In this example we will use Mahout to cluster the data into corresponding class buckets. +A time series of control charts needs to be clustered into their close knit groups. The data set we use is synthetic and is meant to resemble real world information in an anonymized format. It contains six different classes: Normal, Cyclic, Increasing trend, Decreasing trend, Upward shift, Downward shift. In this example we will use Mahout to cluster the data into corresponding class buckets. *For the sake of simplicity, we won't use a cluster in this example, but instead show you the commands to run the clustering examples locally with Hadoop*. ## Setup -We need to do some initial setup before we are able to run the example. +We need to do some initial setup before we are able to run the example. 1. Start out by downloading the dataset to be clustered from the UCI Machine Learning Repository: [http://archive.ics.uci.edu/ml/databases/synthetic_control/synthetic_control.data](http://archive.ics.uci.edu/ml/databases/synthetic_control/synthetic_control.data). @@ -51,4 +51,3 @@ Depending on the clustering algorithm you want to run, the following commands ca bin/mahout org.apache.mahout.clustering.syntheticcontrol.fuzzykmeans.Job The clustering output will be produced in the *output* directory. The output data points are in vector format. In order to read/analyze the output, you can use the [clusterdump](/users/clustering/cluster-dumper.html) utility provided by Mahout. - diff --git a/website/Map_Reduce_Folder/clustering/clusteringyourdata.md b/website/Map_Reduce_Folder/clustering/clusteringyourdata.md index 8f78cf7791..6e88383454 100644 --- a/website/Map_Reduce_Folder/clustering/clusteringyourdata.md +++ b/website/Map_Reduce_Folder/clustering/clusteringyourdata.md @@ -48,34 +48,34 @@ Mahout has a cluster dumper utility that can be used to retrieve and evaluate yo ## The cluster dumper options are: - --help (-h) Print out help - - --input (-i) input The directory containing Sequence - Files for the Clusters + --help (-h) Print out help - --output (-o) output The output file. If not specified, + --input (-i) input The directory containing Sequence + Files for the Clusters + + --output (-o) output The output file. If not specified, dumps to the console. --outputFormat (-of) outputFormat The optional output format to write - the results as. Options: TEXT, CSV, or GRAPH_ML + the results as. Options: TEXT, CSV, or GRAPH_ML + + --substring (-b) substring The number of chars of the + asFormatString() to print - --substring (-b) substring The number of chars of the - asFormatString() to print - - --pointsDir (-p) pointsDir The directory containing points - sequence files mapping input vectors to their cluster. If specified, - then the program will output the - points associated with a cluster + --pointsDir (-p) pointsDir The directory containing points + sequence files mapping input vectors to their cluster. If specified, + then the program will output the + points associated with a cluster - --dictionary (-d) dictionary The dictionary file. + --dictionary (-d) dictionary The dictionary file. - --dictionaryType (-dt) dictionaryType The dictionary file type + --dictionaryType (-dt) dictionaryType The dictionary file type (text|sequencefile) --distanceMeasure (-dm) distanceMeasure The classname of the DistanceMeasure. - Default is SquaredEuclidean. + Default is SquaredEuclidean. - --numWords (-n) numWords The number of top terms to print + --numWords (-n) numWords The number of top terms to print --tempDir tempDir Intermediate output directory @@ -85,7 +85,7 @@ Mahout has a cluster dumper utility that can be used to retrieve and evaluate yo --evaluate (-e) Run ClusterEvaluator and CDbwEvaluator over the input. The output will be appended to the rest of - the output at the end. + the output at the end. More information on using clusterdump utility can be found [here](cluster-dumper.html) @@ -103,7 +103,7 @@ centroid adjusted by the log_2 of the number of clusters. This can be compared to the squared magnitude of the original data or the squared deviation from the centroid for all of the data. The idea is that you are changing the representation of the data by allocating some of the bits in -your original representation to represent which cluster each point is in. +your original representation to represent which cluster each point is in. If those bits aren't made up by the residue being small then your clustering is making a bad trade-off. @@ -124,4 +124,3 @@ Unfortunately, it is pretty easy to kid yourself into thinking your system is working using this kind of inspection. The problem is that we are too good at seeing (making up) patterns. {quote} - diff --git a/website/Map_Reduce_Folder/clustering/expectation-maximization.md b/website/Map_Reduce_Folder/clustering/expectation-maximization.md index da750b877f..fa683f8684 100644 --- a/website/Map_Reduce_Folder/clustering/expectation-maximization.md +++ b/website/Map_Reduce_Folder/clustering/expectation-maximization.md @@ -61,4 +61,3 @@ exponentially small influence. ## Map/Reduce Implementation - diff --git a/website/Map_Reduce_Folder/clustering/fuzzy-k-means-commandline.md b/website/Map_Reduce_Folder/clustering/fuzzy-k-means-commandline.md index d78b8f10be..8e15ed9a67 100644 --- a/website/Map_Reduce_Folder/clustering/fuzzy-k-means-commandline.md +++ b/website/Map_Reduce_Folder/clustering/fuzzy-k-means-commandline.md @@ -30,7 +30,7 @@ job will be mahout-core-0.3.job ## Testing it on one single machine w/o cluster * Put the data: cp testdata -* Run the Job: +* Run the Job: ./bin/mahout fkmeans -i testdata @@ -40,7 +40,7 @@ job will be mahout-core-0.3.job * (As needed) Start up Hadoop: $HADOOP_HOME/bin/start-all.sh * Put the data: $HADOOP_HOME/bin/hadoop fs -put testdata -* Run the Job: +* Run the Job: export HADOOP_HOME= export HADOOP_CONF_DIR=$HADOOP_HOME/conf @@ -52,47 +52,45 @@ to view all outputs. # Command line options - --input (-i) input Path to job input directory. - Must be a SequenceFile of - VectorWritable - --clusters (-c) clusters The input centroids, as Vectors. - Must be a SequenceFile of - Writable, Cluster/Canopy. If k - is also specified, then a random - set of vectors will be selected - and written out to this path - first - --output (-o) output The directory pathname for - output. - --distanceMeasure (-dm) distanceMeasure The classname of the - DistanceMeasure. Default is - SquaredEuclidean - --convergenceDelta (-cd) convergenceDelta The convergence delta value. - Default is 0.5 - --maxIter (-x) maxIter The maximum number of - iterations. - --k (-k) k The k in k-Means. If specified, - then a random selection of k + --input (-i) input Path to job input directory. + Must be a SequenceFile of + VectorWritable + --clusters (-c) clusters The input centroids, as Vectors. + Must be a SequenceFile of + Writable, Cluster/Canopy. If k + is also specified, then a random + set of vectors will be selected + and written out to this path + first + --output (-o) output The directory pathname for + output. + --distanceMeasure (-dm) distanceMeasure The classname of the + DistanceMeasure. Default is + SquaredEuclidean + --convergenceDelta (-cd) convergenceDelta The convergence delta value. + Default is 0.5 + --maxIter (-x) maxIter The maximum number of + iterations. + --k (-k) k The k in k-Means. If specified, + then a random selection of k Vectors will be chosen as the - Centroid and written to the - clusters input path. - --m (-m) m coefficient normalization - factor, must be greater than 1 - --overwrite (-ow) If present, overwrite the output - directory before running job - --help (-h) Print out help - --numMap (-u) numMap The number of map tasks. - Defaults to 10 - --maxRed (-r) maxRed The number of reduce tasks. - Defaults to 2 - --emitMostLikely (-e) emitMostLikely True if clustering should emit - the most likely point only, - false for threshold clustering. - Default is true - --threshold (-t) threshold The pdf threshold used for - cluster determination. Default - is 0 - --clustering (-cl) If present, run clustering after - the iterations have taken place - - + Centroid and written to the + clusters input path. + --m (-m) m coefficient normalization + factor, must be greater than 1 + --overwrite (-ow) If present, overwrite the output + directory before running job + --help (-h) Print out help + --numMap (-u) numMap The number of map tasks. + Defaults to 10 + --maxRed (-r) maxRed The number of reduce tasks. + Defaults to 2 + --emitMostLikely (-e) emitMostLikely True if clustering should emit + the most likely point only, + false for threshold clustering. + Default is true + --threshold (-t) threshold The pdf threshold used for + cluster determination. Default + is 0 + --clustering (-cl) If present, run clustering after + the iterations have taken place diff --git a/website/Map_Reduce_Folder/clustering/fuzzy-k-means.md b/website/Map_Reduce_Folder/clustering/fuzzy-k-means.md index d19ad0ff68..4826b5d827 100644 --- a/website/Map_Reduce_Folder/clustering/fuzzy-k-means.md +++ b/website/Map_Reduce_Folder/clustering/fuzzy-k-means.md @@ -38,8 +38,8 @@ Similar to K-Means, the program doesn't modify the input directories. And for every iteration, the cluster output is stored in a directory cluster-N. The code has set number of reduce tasks equal to number of map tasks. So, those many part-0 - - + + Files are created in clusterN directory. The code uses driver/mapper/combiner/reducer as follows: @@ -77,7 +77,7 @@ converged clusters with 'Vn' clusterId. The Fuzzy k-Means clustering algorithm may be run using a command-line invocation on FuzzyKMeansDriver.main or by making a Java call to -FuzzyKMeansDriver.run(). +FuzzyKMeansDriver.run(). Invocation using the command line takes the form: @@ -144,7 +144,7 @@ SequenceFile(IntWritable, WeightedVectorWritable). The IntWritable _key_ is the clusterId. The WeightedVectorWritable _value_ is a bean containing a double _weight_ and a VectorWritable _vector_ where the weights are computed as 1/(1+distance) where the distance is between the cluster center -and the vector using the chosen DistanceMeasure. +and the vector using the chosen DistanceMeasure. # Examples @@ -165,7 +165,7 @@ The points are generated as follows: sd=0.1 In the first image, the points are plotted and the 3-sigma boundaries of -their generator are superimposed. +their generator are superimposed. ![fuzzy](../../images/SampleData.png) diff --git a/website/Map_Reduce_Folder/clustering/hierarchical-clustering.md b/website/Map_Reduce_Folder/clustering/hierarchical-clustering.md index cd9a996172..1758839eea 100644 --- a/website/Map_Reduce_Folder/clustering/hierarchical-clustering.md +++ b/website/Map_Reduce_Folder/clustering/hierarchical-clustering.md @@ -10,8 +10,7 @@ Hierarchical clustering is the process or finding bigger clusters, and also the smaller clusters inside the bigger clusters. In Apache Mahout, separate algorithms can be used for finding clusters at -different levels. +different levels. See [Top Down Clustering](https://cwiki.apache.org/confluence/display/MAHOUT/Top+Down+Clustering) . - diff --git a/website/Map_Reduce_Folder/clustering/k-means-commandline.md b/website/Map_Reduce_Folder/clustering/k-means-commandline.md index 7c89ed6dfb..2d1394bddc 100644 --- a/website/Map_Reduce_Folder/clustering/k-means-commandline.md +++ b/website/Map_Reduce_Folder/clustering/k-means-commandline.md @@ -10,7 +10,7 @@ redirect_from: # kMeans commandline introduction This quick start page describes how to run the kMeans clustering algorithm -on a Hadoop cluster. +on a Hadoop cluster. # Steps @@ -37,7 +37,7 @@ job will be mahout-core-0.3.job ## Testing it on one single machine w/o cluster * Put the data: cp testdata -* Run the Job: +* Run the Job: ./bin/mahout kmeans -i testdata -o output -c clusters -dm org.apache.mahout.common.distance.CosineDistanceMeasure -x 5 -ow -cd 1 -k @@ -49,7 +49,7 @@ org.apache.mahout.common.distance.CosineDistanceMeasure -x 5 -ow -cd 1 -k * (As needed) Start up Hadoop: $HADOOP_HOME/bin/start-all.sh * Put the data: $HADOOP_HOME/bin/hadoop fs -put testdata -* Run the Job: +* Run the Job: export HADOOP_HOME= export HADOOP_CONF_DIR=$HADOOP_HOME/conf @@ -61,35 +61,34 @@ to view all outputs. # Command line options - --input (-i) input Path to job input directory. - Must be a SequenceFile of - VectorWritable - --clusters (-c) clusters The input centroids, as Vectors. - Must be a SequenceFile of - Writable, Cluster/Canopy. If k - is also specified, then a random - set of vectors will be selected - and written out to this path - first - --output (-o) output The directory pathname for - output. - --distanceMeasure (-dm) distanceMeasure The classname of the - DistanceMeasure. Default is - SquaredEuclidean - --convergenceDelta (-cd) convergenceDelta The convergence delta value. - Default is 0.5 - --maxIter (-x) maxIter The maximum number of - iterations. - --maxRed (-r) maxRed The number of reduce tasks. - Defaults to 2 - --k (-k) k The k in k-Means. If specified, - then a random selection of k - Vectors will be chosen as the - Centroid and written to the - clusters input path. - --overwrite (-ow) If present, overwrite the output - directory before running job - --help (-h) Print out help - --clustering (-cl) If present, run clustering after - the iterations have taken place - + --input (-i) input Path to job input directory. + Must be a SequenceFile of + VectorWritable + --clusters (-c) clusters The input centroids, as Vectors. + Must be a SequenceFile of + Writable, Cluster/Canopy. If k + is also specified, then a random + set of vectors will be selected + and written out to this path + first + --output (-o) output The directory pathname for + output. + --distanceMeasure (-dm) distanceMeasure The classname of the + DistanceMeasure. Default is + SquaredEuclidean + --convergenceDelta (-cd) convergenceDelta The convergence delta value. + Default is 0.5 + --maxIter (-x) maxIter The maximum number of + iterations. + --maxRed (-r) maxRed The number of reduce tasks. + Defaults to 2 + --k (-k) k The k in k-Means. If specified, + then a random selection of k + Vectors will be chosen as the + Centroid and written to the + clusters input path. + --overwrite (-ow) If present, overwrite the output + directory before running job + --help (-h) Print out help + --clustering (-cl) If present, run clustering after + the iterations have taken place diff --git a/website/Map_Reduce_Folder/clustering/latent-dirichlet-allocation.md b/website/Map_Reduce_Folder/clustering/latent-dirichlet-allocation.md index 842c7069f8..dbc27ed2a2 100644 --- a/website/Map_Reduce_Folder/clustering/latent-dirichlet-allocation.md +++ b/website/Map_Reduce_Folder/clustering/latent-dirichlet-allocation.md @@ -13,7 +13,7 @@ Latent Dirichlet Allocation (Blei et al, 2003) is a powerful learning algorithm for automatically and jointly clustering words into "topics" and documents into mixtures of topics. It has been successfully applied to model change in scientific fields over time (Griffiths and Steyvers, 2004; -Hall, et al. 2008). +Hall, et al. 2008). A topic model is, roughly, a hierarchical Bayesian model that associates with each document a probability distribution over "topics", which are in @@ -98,7 +98,7 @@ test_set_percentage>0> \ Topic smoothing should generally be about 50/K, where K is the number of topics. The number of words in the vocabulary can be an upper bound, though -it shouldn't be too high (for memory concerns). +it shouldn't be too high (for memory concerns). Choosing the number of topics is more art than science, and it's recommended that you try several values. @@ -124,7 +124,7 @@ An example is located in mahout/examples/bin/build-reuters.sh. The script automatically downloads the Reuters-21578 corpus, builds a Lucene index and converts the Lucene index to vectors. By uncommenting the last two lines you can then cause it to run LDA on the vectors and finally print the -resultant topics to the console. +resultant topics to the console. To adapt the example yourself, you should note that Lucene has specialized support for Reuters, and that building your own index will require some @@ -140,7 +140,7 @@ document, inferring the posterior probability of each topic for each word in each document. We then take the sufficient statistics and emit them in the form of (log) pseudo-counts for each word in each topic. The M-Step is simply to sum these together and (log) normalize them so that we have a -distribution over the entire vocabulary of the corpus for each topic. +distribution over the entire vocabulary of the corpus for each topic. In implementation, the E-Step is implemented in the Map, and the M-Step is executed in the reduce step, with the final normalization happening as a diff --git a/website/Map_Reduce_Folder/clustering/lda-commandline.md b/website/Map_Reduce_Folder/clustering/lda-commandline.md index f5c135e7c2..6c030d2848 100644 --- a/website/Map_Reduce_Folder/clustering/lda-commandline.md +++ b/website/Map_Reduce_Folder/clustering/lda-commandline.md @@ -9,7 +9,7 @@ redirect_from: # Running Latent Dirichlet Allocation (algorithm) from the Command Line [Since Mahout v0.6](https://issues.apache.org/jira/browse/MAHOUT-897) - lda has been implemented as Collapsed Variable Bayes (cvb). + lda has been implemented as Collapsed Variable Bayes (cvb). Mahout's LDA can be launched from the same command line invocation whether you are running on a single machine in stand-alone mode or on a larger @@ -34,7 +34,7 @@ job will be mahout-core-0.3.job ## Testing it on one single machine w/o cluster * Put the data: cp testdata -* Run the Job: +* Run the Job: ./bin/mahout cvb -i testdata @@ -44,7 +44,7 @@ job will be mahout-core-0.3.job * (As needed) Start up Hadoop: $HADOOP_HOME/bin/start-all.sh * Put the data: $HADOOP_HOME/bin/hadoop fs -put testdata -* Run the Job: +* Run the Job: export HADOOP_HOME= export HADOOP_CONF_DIR=$HADOOP_HOME/conf @@ -56,29 +56,28 @@ to view all outputs. # Command line options from Mahout cvb version 0.8 - mahout cvb -h - --input (-i) input Path to job input directory. - --output (-o) output The directory pathname for output. - --maxIter (-x) maxIter The maximum number of iterations. - --convergenceDelta (-cd) convergenceDelta The convergence delta value - --overwrite (-ow) If present, overwrite the output directory before running job - --num_topics (-k) num_topics Number of topics to learn - --num_terms (-nt) num_terms Vocabulary size - --doc_topic_smoothing (-a) doc_topic_smoothing Smoothing for document/topic distribution - --term_topic_smoothing (-e) term_topic_smoothing Smoothing for topic/term distribution - --dictionary (-dict) dictionary Path to term-dictionary file(s) (glob expression supported) - --doc_topic_output (-dt) doc_topic_output Output path for the training doc/topic distribution - --topic_model_temp_dir (-mt) topic_model_temp_dir Path to intermediate model path (useful for restarting) - --iteration_block_size (-block) iteration_block_size Number of iterations per perplexity check - --random_seed (-seed) random_seed Random seed - --test_set_fraction (-tf) test_set_fraction Fraction of data to hold out for testing - --num_train_threads (-ntt) num_train_threads number of threads per mapper to train with - --num_update_threads (-nut) num_update_threads number of threads per mapper to update the model with - --max_doc_topic_iters (-mipd) max_doc_topic_iters max number of iterations per doc for p(topic|doc) learning - --num_reduce_tasks num_reduce_tasks number of reducers to use during model estimation - --backfill_perplexity enable backfilling of missing perplexity values - --help (-h) Print out help - --tempDir tempDir Intermediate output directory - --startPhase startPhase First phase to run + mahout cvb -h + --input (-i) input Path to job input directory. + --output (-o) output The directory pathname for output. + --maxIter (-x) maxIter The maximum number of iterations. + --convergenceDelta (-cd) convergenceDelta The convergence delta value + --overwrite (-ow) If present, overwrite the output directory before running job + --num_topics (-k) num_topics Number of topics to learn + --num_terms (-nt) num_terms Vocabulary size + --doc_topic_smoothing (-a) doc_topic_smoothing Smoothing for document/topic distribution + --term_topic_smoothing (-e) term_topic_smoothing Smoothing for topic/term distribution + --dictionary (-dict) dictionary Path to term-dictionary file(s) (glob expression supported) + --doc_topic_output (-dt) doc_topic_output Output path for the training doc/topic distribution + --topic_model_temp_dir (-mt) topic_model_temp_dir Path to intermediate model path (useful for restarting) + --iteration_block_size (-block) iteration_block_size Number of iterations per perplexity check + --random_seed (-seed) random_seed Random seed + --test_set_fraction (-tf) test_set_fraction Fraction of data to hold out for testing + --num_train_threads (-ntt) num_train_threads number of threads per mapper to train with + --num_update_threads (-nut) num_update_threads number of threads per mapper to update the model with + --max_doc_topic_iters (-mipd) max_doc_topic_iters max number of iterations per doc for p(topic|doc) learning + --num_reduce_tasks num_reduce_tasks number of reducers to use during model estimation + --backfill_perplexity enable backfilling of missing perplexity values + --help (-h) Print out help + --tempDir tempDir Intermediate output directory + --startPhase startPhase First phase to run --endPhase endPhase Last phase to run - diff --git a/website/Map_Reduce_Folder/clustering/llr---log-likelihood-ratio.md b/website/Map_Reduce_Folder/clustering/llr---log-likelihood-ratio.md index fcb4968366..2141267c98 100644 --- a/website/Map_Reduce_Folder/clustering/llr---log-likelihood-ratio.md +++ b/website/Map_Reduce_Folder/clustering/llr---log-likelihood-ratio.md @@ -26,11 +26,11 @@ positive and false negative rates. The LLR is typically far more suitable in the case of small than many other measures such as Pearson's correlation, Pearson's chi squared statistic or z statistics. The LLR as stated does not, however, make any use of rating data which can limit its -applicability in problems such as the Netflix competition. +applicability in problems such as the Netflix competition. The actual value of the LLR is not usually very helpful other than as a way of ordering pairs of items. As such, it is often used to determine a -sparse set of coefficients to be estimated by other means such as TF-IDF. +sparse set of coefficients to be estimated by other means such as TF-IDF. Since the actual estimation of these coefficients can be done in a way that is independent of the training data such as by general corpus statistics, and since the ordering imposed by the LLR is relatively robust to counting @@ -38,10 +38,8 @@ fluctuation, this technique can provide very strong results in very sparse problems where the potential number of features vastly out-numbers the number of training examples and where features are highly interdependent. - See Also: + See Also: * [Blog post "surprise and coincidence"](http://tdunning.blogspot.com/2008/03/surprise-and-coincidence.html) * [G-Test](http://en.wikipedia.org/wiki/G-test) * [Likelihood Ratio Test](http://en.wikipedia.org/wiki/Likelihood-ratio_test) - - diff --git a/website/Map_Reduce_Folder/clustering/streaming-k-means.md b/website/Map_Reduce_Folder/clustering/streaming-k-means.md index 79fea5a394..28148d8d53 100644 --- a/website/Map_Reduce_Folder/clustering/streaming-k-means.md +++ b/website/Map_Reduce_Folder/clustering/streaming-k-means.md @@ -6,18 +6,18 @@ redirect_from: - /documentation/users/clustering/streaming-k-means.html --- -# *StreamingKMeans* algorithm +# *StreamingKMeans* algorithm The *StreamingKMeans* algorithm is a variant of Algorithm 1 from [Shindler et al][1] and consists of two steps: - 1. Streaming step - 2. BallKMeans step. + 1. Streaming step + 2. BallKMeans step. -The streaming step is a randomized algorithm that makes one pass through the data and -produces as many centroids as it determines is optimal. This step can be viewed as -a preparatory dimensionality reduction. If the size of the data stream is *n* and the -expected number of clusters is *k*, the streaming step will produce roughly *k\*log(n)* -clusters that will be passed on to the BallKMeans step which will further reduce the +The streaming step is a randomized algorithm that makes one pass through the data and +produces as many centroids as it determines is optimal. This step can be viewed as +a preparatory dimensionality reduction. If the size of the data stream is *n* and the +expected number of clusters is *k*, the streaming step will produce roughly *k\*log(n)* +clusters that will be passed on to the BallKMeans step which will further reduce the number of clusters down to *k*. BallKMeans is a randomized Lloyd-type algorithm that has been studied in detail, see [Ostrovsky et al][2]. @@ -27,35 +27,35 @@ has been studied in detail, see [Ostrovsky et al][2]. ### Overview -The streaming step is a derivative of the streaming -portion of Algorithm 1 in [Shindler et al][1]. The main difference between the two is that -Algorithm 1 of [Shindler et al][1] assumes -the knowledge of the size of the data stream and uses it to set a key parameter -for the algorithm. More precisely, the initial *distanceCutoff* (defined below), which is -denoted by *f* in [Shindler et al][1], is set to *1/(k(1+log(n))*. The *distanceCutoff* influences the number of clusters that the algorithm -will produce. -In contrast, Mahout implementation does not require the knowledge of the size of the -data stream. Instead, it dynamically re-evaluates the parameters that depend on the size -of the data stream at runtime as more and more data is processed. In particular, -the parameter *numClusters* (defined below) changes its value as the data is processed. +The streaming step is a derivative of the streaming +portion of Algorithm 1 in [Shindler et al][1]. The main difference between the two is that +Algorithm 1 of [Shindler et al][1] assumes +the knowledge of the size of the data stream and uses it to set a key parameter +for the algorithm. More precisely, the initial *distanceCutoff* (defined below), which is +denoted by *f* in [Shindler et al][1], is set to *1/(k(1+log(n))*. The *distanceCutoff* influences the number of clusters that the algorithm +will produce. +In contrast, Mahout implementation does not require the knowledge of the size of the +data stream. Instead, it dynamically re-evaluates the parameters that depend on the size +of the data stream at runtime as more and more data is processed. In particular, +the parameter *numClusters* (defined below) changes its value as the data is processed. ###Parameters - - **numClusters** (int): Conceptually, *numClusters* represents the algorithm's guess at the optimal -number of clusters it is shooting for. In particular, *numClusters* will increase at run -time as more and more data is processed. Note that •numClusters• is not the number of clusters that the algorithm will produce. Also, *numClusters* should not be set to the final number of clusters that we expect to receive as the output of *StreamingKMeans*. - - **distanceCutoff** (double): a parameter representing the value of the distance between a point and + - **numClusters** (int): Conceptually, *numClusters* represents the algorithm's guess at the optimal +number of clusters it is shooting for. In particular, *numClusters* will increase at run +time as more and more data is processed. Note that •numClusters• is not the number of clusters that the algorithm will produce. Also, *numClusters* should not be set to the final number of clusters that we expect to receive as the output of *StreamingKMeans*. + - **distanceCutoff** (double): a parameter representing the value of the distance between a point and its closest centroid after which -the new point will definitely be assigned to a new cluster. *distanceCutoff* can be thought -of as an estimate of the variable *f* from Shindler et al. The default initial value for -*distanceCutoff* is *1.0/numClusters* and *distanceCutoff* grows as a geometric progression with -common ratio *beta* (see below). - - **beta** (double): a constant parameter that controls the growth of *distanceCutoff*. If the initial setting of *distanceCutoff* is *d0*, *distanceCutoff* will grow as the geometric progression with initial term *d0* and common ratio *beta*. The default value for *beta* is 1.3. - - **clusterLogFactor** (double): a constant parameter such that *clusterLogFactor* *log(numProcessedPoints)* is the runtime estimate of the number of clusters to be produced by the streaming step. If the final number of clusters (that we expect *StreamingKMeans* to output) is *k*, *clusterLogFactor* can be set to *k*. - - **clusterOvershoot** (double): a constant multiplicative slack factor that slows down the collapsing of clusters. The default value is 2. +the new point will definitely be assigned to a new cluster. *distanceCutoff* can be thought +of as an estimate of the variable *f* from Shindler et al. The default initial value for +*distanceCutoff* is *1.0/numClusters* and *distanceCutoff* grows as a geometric progression with +common ratio *beta* (see below). + - **beta** (double): a constant parameter that controls the growth of *distanceCutoff*. If the initial setting of *distanceCutoff* is *d0*, *distanceCutoff* will grow as the geometric progression with initial term *d0* and common ratio *beta*. The default value for *beta* is 1.3. + - **clusterLogFactor** (double): a constant parameter such that *clusterLogFactor* *log(numProcessedPoints)* is the runtime estimate of the number of clusters to be produced by the streaming step. If the final number of clusters (that we expect *StreamingKMeans* to output) is *k*, *clusterLogFactor* can be set to *k*. + - **clusterOvershoot** (double): a constant multiplicative slack factor that slows down the collapsing of clusters. The default value is 2. -###Algorithm +###Algorithm The algorithm processes the data one-by-one and makes only one pass through the data. The first point from the data stream will form the centroid of the first cluster (this designation may change as more points are processed). Suppose there are *r* clusters at one point and a new point *p* is being processed. The new point can either be added to one of the existing *r* clusters or become a new cluster. To decide: @@ -63,7 +63,7 @@ The first point from the data stream will form the centroid of the first cluster - let *c* be the closest cluster to point *p* - let *d* be the distance between *c* and *p* - if *d > distanceCutoff*, create a new cluster from *p* (*p* is too far away from the clusters to be part of any one of them) - - else (*d <= distanceCutoff*), create a new cluster with probability *d / distanceCutoff* (the probability of creating a new cluster increases as *d* increases). + - else (*d <= distanceCutoff*), create a new cluster with probability *d / distanceCutoff* (the probability of creating a new cluster increases as *d* increases). There will be either *r* or *r+1* clusters after processing a new point. @@ -73,12 +73,12 @@ are treated as data points and are re-clustered (collapsed). This tends to make ## BallKMeans step --- ### Overview -The algorithm is a Lloyd-type algorithm that takes a set of weighted vectors and returns k centroids, see [Ostrovsky et al][2] for details. The algorithm has two stages: - - 1. Seeding - 2. Ball k-means +The algorithm is a Lloyd-type algorithm that takes a set of weighted vectors and returns k centroids, see [Ostrovsky et al][2] for details. The algorithm has two stages: + + 1. Seeding + 2. Ball k-means -The seeding stage is an initial guess of where the centroids should be. The initial guess is improved using the ball k-means stage. +The seeding stage is an initial guess of where the centroids should be. The initial guess is improved using the ball k-means stage. ### Parameters @@ -97,71 +97,71 @@ The seeding stage is an initial guess of where the centroids should be. The init * **numRuns** (int): This is the number of runs to perform. The solution of lowest cost is returned. The default is 1 run. ###Algorithm -The algorithm can be instructed to take multiple independent runs (using the *numRuns* parameter) and the algorithm will select the best solution (i.e., the one with the lowest cost). In practice, one run is sufficient to find a good solution. +The algorithm can be instructed to take multiple independent runs (using the *numRuns* parameter) and the algorithm will select the best solution (i.e., the one with the lowest cost). In practice, one run is sufficient to find a good solution. Each run operates as follows: a seeding procedure is used to select k centroids, and then ball k-means is run iteratively to refine the solution. -The seeding procedure can be set to either 'uniformly at random' or 'k-means++' using *kMeansPlusPlusInit* boolean variable. Seeding with k-means++ involves more computation but offers better results in practice. - +The seeding procedure can be set to either 'uniformly at random' or 'k-means++' using *kMeansPlusPlusInit* boolean variable. Seeding with k-means++ involves more computation but offers better results in practice. + Each iteration of ball k-means runs as follows: 1. Clusters are formed by assigning each datapoint to the nearest centroid -2. The centers of mass of the trimmed clusters (see *trimFraction* parameter above) become the new centroids +2. The centers of mass of the trimmed clusters (see *trimFraction* parameter above) become the new centroids The data may be partitioned into a test set and a training set (see *testProbability*). The seeding procedure and ball k-means run on the training set. The cost is computed on the test set. -##Usage of *StreamingKMeans* - - bin/mahout streamingkmeans - -i - -o - -ow - -k - -km - -e - -mi - -tf - -ri - -iw - -testp - -nbkm - -dm - -sc - -np - -s - -rskm - -xm - -h - --tempDir - --startPhase - --endPhase +##Usage of *StreamingKMeans* + + bin/mahout streamingkmeans + -i + -o + -ow + -k + -km + -e + -mi + -tf + -ri + -iw + -testp + -nbkm + -dm + -sc + -np + -s + -rskm + -xm + -h + --tempDir + --startPhase + --endPhase ###Details on Job-Specific Options: - - * `--input (-i) `: Path to job input directory. - * `--output (-o) `: The directory pathname for output. + + * `--input (-i) `: Path to job input directory. + * `--output (-o) `: The directory pathname for output. * `--overwrite (-ow)`: If present, overwrite the output directory before running job. - * `--numClusters (-k) `: The k in k-Means. Approximately this many clusters will be generated. - * `--estimatedNumMapClusters (-km) `: The estimated number of clusters to use for the Map phase of the job when running StreamingKMeans. This should be around k \* log(n), where k is the final number of clusters and n is the total number of data points to cluster. - * `--estimatedDistanceCutoff (-e) `: The initial estimated distance cutoff between two points for forming new clusters. If no value is given, it's estimated from the data set - * `--maxNumIterations (-mi) `: The maximum number of iterations to run for the BallKMeans algorithm used by the reducer. If no value is given, defaults to 10. - * `--trimFraction (-tf) `: The 'ball' aspect of ball k-means means that only the closest points to the centroid will actually be used for updating. The fraction of the points to be used is those points whose distance to the center is within trimFraction \* distance to the closest other center. If no value is given, defaults to 0.9. + * `--numClusters (-k) `: The k in k-Means. Approximately this many clusters will be generated. + * `--estimatedNumMapClusters (-km) `: The estimated number of clusters to use for the Map phase of the job when running StreamingKMeans. This should be around k \* log(n), where k is the final number of clusters and n is the total number of data points to cluster. + * `--estimatedDistanceCutoff (-e) `: The initial estimated distance cutoff between two points for forming new clusters. If no value is given, it's estimated from the data set + * `--maxNumIterations (-mi) `: The maximum number of iterations to run for the BallKMeans algorithm used by the reducer. If no value is given, defaults to 10. + * `--trimFraction (-tf) `: The 'ball' aspect of ball k-means means that only the closest points to the centroid will actually be used for updating. The fraction of the points to be used is those points whose distance to the center is within trimFraction \* distance to the closest other center. If no value is given, defaults to 0.9. * `--randomInit` (`-ri`) Whether to use k-means++ initialization or random initialization of the seed centroids. Essentially, k-means++ provides better clusters, but takes longer, whereas random initialization takes less time, but produces worse clusters, and tends to fail more often and needs multiple runs to compare to k-means++. If set, uses the random initialization. - * `--ignoreWeights (-iw)`: Whether to correct the weights of the centroids after the clustering is done. The weights end up being wrong because of the trimFraction and possible train/test splits. In some cases, especially in a pipeline, having an accurate count of the weights is useful. If set, ignores the final weights. - * `--testProbability (-testp) `: A double value between 0 and 1 that represents the percentage of points to be used for 'testing' different clustering runs in the final BallKMeans step. If no value is given, defaults to 0.1 - * `--numBallKMeansRuns (-nbkm) `: Number of BallKMeans runs to use at the end to try to cluster the points. If no value is given, defaults to 4 - * `--distanceMeasure (-dm) `: The classname of the DistanceMeasure. Default is SquaredEuclidean. - * `--searcherClass (-sc) `: The type of searcher to be used when performing nearest neighbor searches. Defaults to ProjectionSearch. - * `--numProjections (-np) `: The number of projections considered in estimating the distances between vectors. Only used when the distance measure requested is either ProjectionSearch or FastProjectionSearch. If no value is given, defaults to 3. - * `--searchSize (-s) `: In more efficient searches (non BruteSearch), not all distances are calculated for determining the nearest neighbors. The number of elements whose distances from the query vector is actually computer is proportional to searchSize. If no value is given, defaults to 1. - * `--reduceStreamingKMeans (-rskm)`: There might be too many intermediate clusters from the mapper to fit into memory, so the reducer can run another pass of StreamingKMeans to collapse them down to a fewer clusters. - * `--method (-xm)` method The execution method to use: sequential or mapreduce. Default is mapreduce. - * `-- help (-h)`: Print out help + * `--ignoreWeights (-iw)`: Whether to correct the weights of the centroids after the clustering is done. The weights end up being wrong because of the trimFraction and possible train/test splits. In some cases, especially in a pipeline, having an accurate count of the weights is useful. If set, ignores the final weights. + * `--testProbability (-testp) `: A double value between 0 and 1 that represents the percentage of points to be used for 'testing' different clustering runs in the final BallKMeans step. If no value is given, defaults to 0.1 + * `--numBallKMeansRuns (-nbkm) `: Number of BallKMeans runs to use at the end to try to cluster the points. If no value is given, defaults to 4 + * `--distanceMeasure (-dm) `: The classname of the DistanceMeasure. Default is SquaredEuclidean. + * `--searcherClass (-sc) `: The type of searcher to be used when performing nearest neighbor searches. Defaults to ProjectionSearch. + * `--numProjections (-np) `: The number of projections considered in estimating the distances between vectors. Only used when the distance measure requested is either ProjectionSearch or FastProjectionSearch. If no value is given, defaults to 3. + * `--searchSize (-s) `: In more efficient searches (non BruteSearch), not all distances are calculated for determining the nearest neighbors. The number of elements whose distances from the query vector is actually computer is proportional to searchSize. If no value is given, defaults to 1. + * `--reduceStreamingKMeans (-rskm)`: There might be too many intermediate clusters from the mapper to fit into memory, so the reducer can run another pass of StreamingKMeans to collapse them down to a fewer clusters. + * `--method (-xm)` method The execution method to use: sequential or mapreduce. Default is mapreduce. + * `-- help (-h)`: Print out help * `--tempDir `: Intermediate output directory. - * `--startPhase ` First phase to run. - * `--endPhase ` Last phase to run. + * `--startPhase ` First phase to run. + * `--endPhase ` Last phase to run. ##References diff --git a/website/Map_Reduce_Folder/clustering/viewing-results.md b/website/Map_Reduce_Folder/clustering/viewing-results.md index 3e7726d3d6..3f7e71b7d9 100644 --- a/website/Map_Reduce_Folder/clustering/viewing-results.md +++ b/website/Map_Reduce_Folder/clustering/viewing-results.md @@ -40,7 +40,7 @@ Run the following to print out all options: ./solr-clust-n2/out/clusters-2 --dictionary ./solr-clust-n2/dictionary.txt --substring 100 --pointsDir ./solr-clust-n2/out/points/ - + diff --git a/website/Map_Reduce_Folder/clustering/visualizing-sample-clusters.md b/website/Map_Reduce_Folder/clustering/visualizing-sample-clusters.md index 4d81ae4e24..a28afdf442 100644 --- a/website/Map_Reduce_Folder/clustering/visualizing-sample-clusters.md +++ b/website/Map_Reduce_Folder/clustering/visualizing-sample-clusters.md @@ -41,7 +41,7 @@ If you are using Eclipse, just right-click on each of the classes mentioned abov cd $MAHOUT_HOME/examples mvn -q exec:java -Dexec.mainClass=org.apache.mahout.clustering.display.DisplayClustering -You can substitute other names above for *DisplayClustering*. +You can substitute other names above for *DisplayClustering*. Note that some of these programs display the sample points and then superimpose all of the clusters from each iteration. The last iteration's clusters are in diff --git a/website/Map_Reduce_Folder/developers/buildingmahout.md b/website/Map_Reduce_Folder/developers/buildingmahout.md index 7762e60eb7..0b6a813bd4 100644 --- a/website/Map_Reduce_Folder/developers/buildingmahout.md +++ b/website/Map_Reduce_Folder/developers/buildingmahout.md @@ -19,10 +19,10 @@ redirect_from: Checkout the sources from the [Mahout GitHub repository](https://github.com/apache/mahout) either via - + git clone git@github.com:apache/mahout.git or - + git clone https://github.com/apache/mahout.git ## Building From Source @@ -37,14 +37,14 @@ NVIDIA Card (installed with OpenCL drivers alongside usual GPU drivers) Install java 1.7+ in an easily accessible directory (for this example, ~/java/) http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html - + Create a directory ~/apache/ . - + Download apache Maven 3.3.9 and un-tar/gunzip to ~/apache/apache-maven-3.3.9/ . https://maven.apache.org/download.cgi - + Download and un-tar/gunzip Hadoop 2.4.1 to ~/apache/hadoop-2.4.1/ . -https://archive.apache.org/dist/hadoop/common/hadoop-2.4.1/ +https://archive.apache.org/dist/hadoop/common/hadoop-2.4.1/ Download and un-tar/gunzip spark-1.6.3-bin-hadoop2.4 to ~/apache/ . http://spark.apache.org/downloads.html @@ -81,7 +81,7 @@ When building mahout for a spark backend, we need four System Environment variab ``` export MAHOUT_HOME=/home//apache/mahout export HADOOP_HOME=/home//apache/hadoop-2.4.1 - export SPARK_HOME=/home//apache/spark-1.6.3-bin-hadoop2.4 + export SPARK_HOME=/home//apache/spark-1.6.3-bin-hadoop2.4 export JAVA_HOME=/home//java/jdk-1.8.121 ``` @@ -95,7 +95,7 @@ To use all available local cores (again, Spark master need not be running) ``` export MASTER=local[*] ``` -To point to a cluster with spark running: +To point to a cluster with spark running: ``` export MASTER=spark://master.ip.address:7077 ``` @@ -131,7 +131,7 @@ mvn clean install -Pviennacl -DskipTests Apache Maven encourages users to make use of build profiles for selectively building modules. -The command +The command ```bash mvn clean package ``` @@ -139,13 +139,13 @@ mvn clean package Is the basic build command. This default will build the following packages. ```bash -[INFO] Apache Mahout -[INFO] Mahout Core -[INFO] Mahout Engine -[INFO] - Mahout HDFS Support -[INFO] - Mahout Spark Engine -[INFO] Mahout Community -[INFO] - Mahout Spark CLI Drivers +[INFO] Apache Mahout +[INFO] Mahout Core +[INFO] Mahout Engine +[INFO] - Mahout HDFS Support +[INFO] - Mahout Spark Engine +[INFO] Mahout Community +[INFO] - Mahout Spark CLI Drivers ``` The following profiles are available for building optional components: @@ -197,7 +197,7 @@ The following profiles are available for building optional components: #### Example -If you want to build Apache Mahout with ViennaCL OpenMP support but skip the command line Spark drivers you would use this +If you want to build Apache Mahout with ViennaCL OpenMP support but skip the command line Spark drivers you would use this command to build: ```bash @@ -206,13 +206,10 @@ mvn clean package -Pviennacl-omp -DskipCli #### Building Java/Scala Docs -To build the Java/Scala docs use the maven `site` goal and the `docs` profile. +To build the Java/Scala docs use the maven `site` goal and the `docs` profile. Additionally, passing the `-Ddependency.locations.enabled=false` option will skip checking the dependency location and allow a much faster build. - + ```bash mvn clean site -Pall,docs -Ddependency.locations.enabled=false ``` - - - diff --git a/website/Map_Reduce_Folder/developers/developer-resources.md b/website/Map_Reduce_Folder/developers/developer-resources.md index 7b0decf6b9..6433f7b308 100644 --- a/website/Map_Reduce_Folder/developers/developer-resources.md +++ b/website/Map_Reduce_Folder/developers/developer-resources.md @@ -4,7 +4,7 @@ title: Developer Resources redirect_from: - /documentation/developers/developer-resources - /documentation/developers/developer-resources.html - + --- diff --git a/website/Map_Reduce_Folder/developers/github.md b/website/Map_Reduce_Folder/developers/github.md index 5c88966702..78ce347dd6 100644 --- a/website/Map_Reduce_Folder/developers/github.md +++ b/website/Map_Reduce_Folder/developers/github.md @@ -5,22 +5,22 @@ redirect_from: - /documentation/developers/github - /documentation/developers/github.html - + --- # Github Setup and Pull Requests (PRs) # -There are several ways to setup Git for committers and contributors. Contributors can safely setup -Git any way they choose but committers should take extra care since they can push new commits to the master at -Apache and various policies there make backing out mistakes problematic. Therefore all but very small changes should +There are several ways to setup Git for committers and contributors. Contributors can safely setup +Git any way they choose but committers should take extra care since they can push new commits to the master at +Apache and various policies there make backing out mistakes problematic. Therefore all but very small changes should go through a PR, even for committers. To keep the commit history clean take note of the use of --squash below when merging into apache/master. ## Git setup for Committers -This describes setup for one local repo and two remotes. It allows you to push the code on your machine to either your Github repo or to git-wip-us.apache.org. -You will want to fork github's apache/mahout to your own account on github, this will enable Pull Requests of your own. -Cloning this fork locally will set up "origin" to point to your remote fork on github as the default remote. +This describes setup for one local repo and two remotes. It allows you to push the code on your machine to either your Github repo or to git-wip-us.apache.org. +You will want to fork github's apache/mahout to your own account on github, this will enable Pull Requests of your own. +Cloning this fork locally will set up "origin" to point to your remote fork on github as the default remote. So if you perform "git push origin master" it will go to github. To attach to the apache git repo do the following: @@ -45,7 +45,7 @@ Now if you want to experiment with a branch everything, by default, points to yo git commit -a -m "doing some work" git push origin mahout-xxxx # notice pushing to **origin** not **apache** -Once you are ready to commit to the apache remote you can merge and push them directly or better yet create a PR. +Once you are ready to commit to the apache remote you can merge and push them directly or better yet create a PR. ## How to create a PR (committers) @@ -55,52 +55,52 @@ Push your branch to Github: git push origin mahout-xxxx Go to your mahout-xxxx branch on Github. Since you forked it from Github's apache/mahout it will default -any PR to go to apache/master. - -* Click the green "Compare, review, and create pull request" button. -* You can edit the to and from for the PR if it isn't correct. The "base fork" should be apache/mahout unless you are collaborating -separately with one of the committers on the list. The "base" will be master. Don't submit a PR to one of the other -branches unless you know what you are doing. The "head fork" will be your forked repo and the "compare" will be -your mahout-xxxx branch. -* Click the "Create pull request" button and name the request "MAHOUT-XXXX" all caps. +any PR to go to apache/master. + +* Click the green "Compare, review, and create pull request" button. +* You can edit the to and from for the PR if it isn't correct. The "base fork" should be apache/mahout unless you are collaborating +separately with one of the committers on the list. The "base" will be master. Don't submit a PR to one of the other +branches unless you know what you are doing. The "head fork" will be your forked repo and the "compare" will be +your mahout-xxxx branch. +* Click the "Create pull request" button and name the request "MAHOUT-XXXX" all caps. This will connect the comments of the PR to the mailing list and Jira comments. -* From now on the PR lives on github's apache/mahout. You use the commenting UI there. -* If you are looking for a review or sharing with someone else say so in the comments but don't worry about -automated merging of your PR--you will have to do that later. The PR is tied to your branch so you can respond to -comments, make fixes, and commit them from your local repo. They will appear on the PR page and be mirrored to Jira -and the mailing list. +* From now on the PR lives on github's apache/mahout. You use the commenting UI there. +* If you are looking for a review or sharing with someone else say so in the comments but don't worry about +automated merging of your PR--you will have to do that later. The PR is tied to your branch so you can respond to +comments, make fixes, and commit them from your local repo. They will appear on the PR page and be mirrored to Jira +and the mailing list. When you are satisfied and want to push it to Apache's remote repo proceed with **Merging a PR** ## How to create a PR (contributors) -Create pull requests: \[[1]\]. +Create pull requests: \[[1]\]. -Pull requests are made to apache/mahout repository on Github. In the Github UI you should pick the master -branch to target the PR as described for committers. This will be reviewed and commented on so the merge is +Pull requests are made to apache/mahout repository on Github. In the Github UI you should pick the master +branch to target the PR as described for committers. This will be reviewed and commented on so the merge is not automatic. This can be used for discussing a contributions in progress. -## Merging a PR (yours or contributors) +## Merging a PR (yours or contributors) -Start with reading \[[2]\] (merging locally). +Start with reading \[[2]\] (merging locally). -Remember that pull requests are equivalent to a remote github branch with potentially a multitude of commits. -In this case it is recommended to squash remote commit history to have one commit per issue, rather -than merging in a multitude of contributor's commits. In order to do that, as well as close the PR at the +Remember that pull requests are equivalent to a remote github branch with potentially a multitude of commits. +In this case it is recommended to squash remote commit history to have one commit per issue, rather +than merging in a multitude of contributor's commits. In order to do that, as well as close the PR at the same time, it is recommended to use **squash commits**. Merging pull requests are equivalent to a "pull" of a contributor's branch: git checkout master # switch to local master branch git pull apache master # fast-forward to current remote HEAD - git pull --squash https://github.com/cuser/mahout cbranch # merge to master + git pull --squash https://github.com/cuser/mahout cbranch # merge to master --squash ensures all PR history is squashed into single commit, and allows committer to use his/her own -message. Read git help for merge or pull for more information about `--squash` option. In this example we -assume that the contributor's Github handle is "cuser" and the PR branch name is "cbranch". +message. Read git help for merge or pull for more information about `--squash` option. In this example we +assume that the contributor's Github handle is "cuser" and the PR branch name is "cbranch". Next, resolve conflicts, if any, or ask a contributor to rebase on top of master, if PR went out of sync. -If you are ready to merge your own (committer's) PR you probably only need to merge (not pull), since you have a local copy +If you are ready to merge your own (committer's) PR you probably only need to merge (not pull), since you have a local copy that you've been working on. This is the branch that you used to create the PR. git checkout master # switch to local master branch @@ -113,7 +113,7 @@ If everything is fine, you now can commit the squashed request along the lines git commit --author -a -m "MAHOUT-XXXX description closes apache/mahout#ZZ" -MAHOUT-XXXX is all caps and where `ZZ` is the pull request number on apache/mahout repository. Including +MAHOUT-XXXX is all caps and where `ZZ` is the pull request number on apache/mahout repository. Including "closes apache/mahout#ZZ" will close the PR automatically. More information is found here \[[3]\]. Next, push to git-wip-us.a.o: @@ -126,31 +126,31 @@ The PR, once pushed, will get mirrored to github. To update your github version push origin master -*Note on squashing: Since squash discards remote branch history, repeated PRs from the same remote branch are -difficult for merging. The workflow implies that every new PR starts with a new rebased branch. This is more +*Note on squashing: Since squash discards remote branch history, repeated PRs from the same remote branch are +difficult for merging. The workflow implies that every new PR starts with a new rebased branch. This is more important for contributors to know, rather than for committers, because if new PR is not mergeable, github would warn to begin with. Anyway, watch for dupe PRs (based on same source branches). This is a bad practice.* - + ## Closing a PR without committing (for committers) -When we want to reject a PR (close without committing), we can just issue an empty commit on master's HEAD -*without merging the PR*: +When we want to reject a PR (close without committing), we can just issue an empty commit on master's HEAD +*without merging the PR*: git commit --allow-empty -m "closes apache/mahout#ZZ *Won't fix*" git push apache master that should close PR `ZZ` on github mirror without merging and any code modifications in the master repository. -## Apache/github integration features +## Apache/github integration features Read \[[4]\]. Comments and PRs with Mahout issue handles should post to mailing lists and Jira. -Mahout issue handles must be in the form MAHOUT-YYYYY (all capitals). Usually it makes sense to -file a jira issue first, and then create a PR with description - +Mahout issue handles must be in the form MAHOUT-YYYYY (all capitals). Usually it makes sense to +file a jira issue first, and then create a PR with description + MAHOUT-YYYY: -In this case all subsequent comments will automatically be copied to jira without having to mention +In this case all subsequent comments will automatically be copied to jira without having to mention jira issue explicitly in each comment of the PR. diff --git a/website/Map_Reduce_Folder/developers/githubPRs.md b/website/Map_Reduce_Folder/developers/githubPRs.md index 754206d2d5..0383553dd3 100644 --- a/website/Map_Reduce_Folder/developers/githubPRs.md +++ b/website/Map_Reduce_Folder/developers/githubPRs.md @@ -4,7 +4,7 @@ title: redirect_from: - /documentation/developers/githubPRs - /documentation/developers/githubPRs.html - + --- # Handling Github PRs # @@ -14,65 +14,65 @@ Hit the merge button. ## how to create a PR (for contributers) -Read [[1]]. +Read [[1]]. -Pull requests are made to apache/mahout repository on Github. +Pull requests are made to apache/mahout repository on Github. -## merging a PR and closing it (for committers). +## merging a PR and closing it (for committers). -Remember that pull requests are equivalent to a remote branch with potentially a multitude of commits. -In this case it is recommended to squash remote commit history to have one commit per issue, rather -than merging in a multitude of contributer's commits. In order to do that, as well as close the PR at the +Remember that pull requests are equivalent to a remote branch with potentially a multitude of commits. +In this case it is recommended to squash remote commit history to have one commit per issue, rather +than merging in a multitude of contributer's commits. In order to do that, as well as close the PR at the same time, it is recommended to use **squash commits**. Read [[2]] (merging locally). Merging pull requests are equivalent to merging contributor's branch: git checkout master # switch to local master branch git pull apache master # fast-forward to current remote HEAD - git pull --squash https://github.com/cuser/mahout cbranch # merge to master + git pull --squash https://github.com/cuser/mahout cbranch # merge to master -In this example we assume that contributor Github handle is "cuser" and the PR branch name is "cbranch" there. We also -assume that *apache* remote is configured as +In this example we assume that contributor Github handle is "cuser" and the PR branch name is "cbranch" there. We also +assume that *apache* remote is configured as apache https://git-wip-us.apache.org/repos/asf/mahout.git (fetch) apache https://git-wip-us.apache.org/repos/asf/mahout.git (push) -Squash pull ensures all PR history is squashed into single commit. Also, it is not yet committed, even if +Squash pull ensures all PR history is squashed into single commit. Also, it is not yet committed, even if fast forward is possible, so you get chance to change things before committing. At this point resolve conflicts, if any, or ask contributor to rebase on top of master, if PR went out of sync. Also run regular patch checks and change CHANGELOG. -Suppose everything is fine, you now can commit the squashed request +Suppose everything is fine, you now can commit the squashed request git commit -a -edit message to contain "MAHOUT-YYYY description **closes #ZZ**", where ZZ is the pull request number. +edit message to contain "MAHOUT-YYYY description **closes #ZZ**", where ZZ is the pull request number. Including "closes #ZZ" will close PR automatically. More information [[3]]. push apache master (this will require credentials). -Note on squashing: Since squash discards remote branch history, repeated PRs from the same remote branch are -difficult for merging. The workflow implies that every new PR starts with a new rebased branch. This is more +Note on squashing: Since squash discards remote branch history, repeated PRs from the same remote branch are +difficult for merging. The workflow implies that every new PR starts with a new rebased branch. This is more important for contributors to know, rather than for committers, because if new PR is not mergeable, github would warn to begin with. Anyway, watch for dupe PRs (based on same source branches). This is a bad practice. - -## Closing a PR without committing -When we want to reject a PR (close without committing), just do the following commit on master's HEAD -*without merging the PR*: +## Closing a PR without committing + +When we want to reject a PR (close without committing), just do the following commit on master's HEAD +*without merging the PR*: git commit --allow-empty -m "closes #ZZ *Won't fix*" git push apache master that should close PR without merging and any code modifications in the master repository. -## Apache/github integration features +## Apache/github integration features Read [[4]]. Issue handles mentioned in comments and PR name should post to mailing lists and Jira. diff --git a/website/Map_Reduce_Folder/developers/gsoc.md b/website/Map_Reduce_Folder/developers/gsoc.md index 098771987f..20a27f337e 100644 --- a/website/Map_Reduce_Folder/developers/gsoc.md +++ b/website/Map_Reduce_Folder/developers/gsoc.md @@ -4,7 +4,7 @@ title: GSOC redirect_from: - /documentaion/developers/gsoc - /documentaion/developers/gsoc.html - + --- # Google Summer of Code @@ -43,7 +43,7 @@ being selected, not increase them. students who propose to implement 3-5 machine learning algorithms on Map/Reduce, all in a two month period. They NEVER get selected. Be realistic. All successful projects to date follow, more or less, the -following formula: Implement algorithm on Map/Reduce. Write Unit Tests. +following formula: Implement algorithm on Map/Reduce. Write Unit Tests. Do some bigger scale tests. Write 1 or 2 examples. Write Wiki documentation. That's it. Trust us, it takes a summer to do these things. @@ -57,7 +57,7 @@ evaluation will take place privately. about what you are implementing. We care about you contributing to open source. You learn. We learn. Win-win. * Enjoy it! Contributing to open source can open some amazing doors for -your career. +your career. ## References diff --git a/website/Map_Reduce_Folder/developers/how-to-become-a-committer.md b/website/Map_Reduce_Folder/developers/how-to-become-a-committer.md index 61ff3c6bfc..3d340d0d33 100644 --- a/website/Map_Reduce_Folder/developers/how-to-become-a-committer.md +++ b/website/Map_Reduce_Folder/developers/how-to-become-a-committer.md @@ -4,7 +4,7 @@ title: How To Become A Committer redirect_from: - /documentation/developers/how-to-become-a-committer - /documentation/developers/how-to-become-a-committer.html - + --- # How to become a committer diff --git a/website/Map_Reduce_Folder/developers/how-to-release.md b/website/Map_Reduce_Folder/developers/how-to-release.md index 6c5c9f23cb..6ab1d3a374 100644 --- a/website/Map_Reduce_Folder/developers/how-to-release.md +++ b/website/Map_Reduce_Folder/developers/how-to-release.md @@ -5,7 +5,7 @@ redirect_from: - /documentation/developers/how-to-release - /documentation/developers/how-to-release.html - + --- # How To Release Mahout @@ -87,9 +87,9 @@ warnings) * Create the release candidate: `mvn -Pmahout-release,apache-release release:prepare release:perform`; to add credentials for source control use `mvn -Dusername=myusername -Dpassword='mypassword' -Papache-release release:prepare release:perform` * If you have problems authenticating to svn.apache.org, try adding to the command line `-Dusername=USERNAME -Dpassword=PASSWORD` -* If there is an issue first try doing: - 1. `mvn -Dmahout-release,apache-release release:rollback` - 1. followed by `mvn -Dmahout-release,apache-release release:clean` as this will likely save you time and do the right thing. You may also have to rollback the version numbers in the POM files. +* If there is an issue first try doing: + 1. `mvn -Dmahout-release,apache-release release:rollback` + 1. followed by `mvn -Dmahout-release,apache-release release:clean` as this will likely save you time and do the right thing. You may also have to rollback the version numbers in the POM files. 1. _Optional_ If you want to skip test cases while rebuilding, use `mvn -DpreparationGoals="clean compile" release:prepare release:perform` * Review the artifacts, etc. on the Apache Repository (using Sonatype's Nexus application) site: https://repository.apache.org/. You will need to login using your ASF SVN credentials and then browse to the staging area. * Once you have reviewed the artifacts, you will need to "Close" out @@ -107,7 +107,7 @@ release:perform target ``` mvn -Pmahout-release,apache-release versions:set -DnewVersion=PREVIOUS_SNAPSHOT mvn -Pmahout-release,apache-release versions:commit -git commit +git commit git push --delete apache (deletes the remote tag) git tag -d tagname (deletes the local tag) ``` diff --git a/website/Map_Reduce_Folder/developers/how-to-update-the-website.md b/website/Map_Reduce_Folder/developers/how-to-update-the-website.md index 8f2bb824b0..1ec3727f15 100644 --- a/website/Map_Reduce_Folder/developers/how-to-update-the-website.md +++ b/website/Map_Reduce_Folder/developers/how-to-update-the-website.md @@ -12,10 +12,10 @@ Committers and contributors are all welcomed and encouraged to update the Mahout Mahout uses Jekyll to build the website. A [script](https://github.com/apache/mahout/blob/master/website/build_site.sh) is executed by Jenkins after any change is detected in the `master` branch of the official Apache repository ( e.g. after any PR is merged ). As such it is important that you, and any reviewers stage site changes locally before - committing. - + committing. + The process for updating the website is as follows: - + #### Clone the Mahout Git Repository Locally ```git clone http://github.com/apache/mahout``` @@ -24,7 +24,7 @@ See [Building from Source](buildingmahout.html#getting-code) for more details. #### Working with Markdown -Jekyll uses [Kramdown](https://kramdown.gettalong.org/syntax.html) to compile markdown into HTML. +Jekyll uses [Kramdown](https://kramdown.gettalong.org/syntax.html) to compile markdown into HTML. Kramdown syntax is very similar to standard markdown, but with a few subtle diferences, please review to the [Kramdown syntax guide](https://kramdown.gettalong.org/syntax.html). @@ -35,8 +35,8 @@ LaTeX portions are signalled with enclosing `$` characters, e.g. `$$ ... $$` #### Stage changes locally -This step actually can be done _while you are editing_. Then you can see you changes in near-real time -(pending browser refreshes). +This step actually can be done _while you are editing_. Then you can see you changes in near-real time +(pending browser refreshes). In the terminal, @@ -52,10 +52,10 @@ open your favorite browser and make sure your changes look the way you expect th Once you're sure everything is right, you commit your code, push to your github.com account (preferably on a branch other than `trunk` then click "OpenPR"). This process closely follows [How To Contribute- Making Changes](/about/how-to-contribute.html#HowToContribute-MakingChanges) with an exception that for _WEBSITE ONLY_ changes we relax the requirement to open a JIRA ticket. That is to say, small -website changes such as fixing a broken link or typo, do not require a specific JIRA issues, and where you would normally -commit with a message like `MAHOUT-XXXX The thing I did` (where `XXXX` is the assosciated JIRA number), you can instead -simply create a message like `WEBSITE Typos in how-to-update-the-website.md`. There's nothing to stop you from making a -JIRA issue, it simply isn't required. +website changes such as fixing a broken link or typo, do not require a specific JIRA issues, and where you would normally +commit with a message like `MAHOUT-XXXX The thing I did` (where `XXXX` is the assosciated JIRA number), you can instead +simply create a message like `WEBSITE Typos in how-to-update-the-website.md`. There's nothing to stop you from making a +JIRA issue, it simply isn't required. The same goes for when you open a PR (where conventionally one includes the JIRA issue, you can again title `WEBSITE` to indicate there is no JIRA) @@ -66,10 +66,10 @@ A committer will be along shortly to review your changes, please be patient. In other contributors PRs. (Here's a little secret, this is a great way to signal to us that you're interested in becoming a committer too, as PR reviews is a big part of a committer's job). -Once everything is confirmed to be in order, the committer will merge your pull request. +Once everything is confirmed to be in order, the committer will merge your pull request. #### Committers ONLY No further action is needed, this section is here to deliniate from the old CMS system and Jekyll builds of other projects. Jenkins will execute [build_site.sh](https://github.com/apache/mahout/blob/trunk/website/build_site.sh) upon merging. This will build the website and -copy it to the `asf-site` branch, where [mahout.apache.org](http://mahout.apache.org) is served from. +copy it to the `asf-site` branch, where [mahout.apache.org](http://mahout.apache.org) is served from. diff --git a/website/Map_Reduce_Folder/developers/issue-tracker.md b/website/Map_Reduce_Folder/developers/issue-tracker.md index 44000ecabb..e803b650f5 100644 --- a/website/Map_Reduce_Folder/developers/issue-tracker.md +++ b/website/Map_Reduce_Folder/developers/issue-tracker.md @@ -14,7 +14,7 @@ For most changes (apart from trivial stuff) Mahout works according to a review-t This means anything that is to be added is first presented as a patch in the issue tracker. All conversations in the issue tracker are automatically echoed on the developer mailing list and people tend to respond or continue conversations there rather in the issue tracker, so in order to follow an -issue you also have to read to the [mailing lists](/community/mailing-lists.html). +issue you also have to read to the [mailing lists](/community/mailing-lists.html). An issue does not literally have to be an issue. It could be a wish, task, bug report, etc. and it does not have to contain a patch. diff --git a/website/Map_Reduce_Folder/developers/patch-check-list.md b/website/Map_Reduce_Folder/developers/patch-check-list.md index 05c8cc31b2..1ce8efde8c 100644 --- a/website/Map_Reduce_Folder/developers/patch-check-list.md +++ b/website/Map_Reduce_Folder/developers/patch-check-list.md @@ -5,7 +5,7 @@ redirect_from: - /documentation/developers/patch-check-list - /documentation/developers/patch-check-list.html - + --- # Patch Check List @@ -20,11 +20,9 @@ granted? - Make sure you update JIRA by assigning the issue to you so that others know you are working on it. - How is the documentation, especially the javadocs? - - Before committing, make sure you add any new documents to your local Git repo. + - Before committing, make sure you add any new documents to your local Git repo. - Run all unit tests, verify that all tests pass. - + After the above steps are verified and completed, and the contribution is ready to merge, follow the steps in the "Merging a PR" section in: [Handling Github PRs](github.html). - Remember to update the issue status in JIRA when you have completed it. - - diff --git a/website/Map_Reduce_Folder/developers/thirdparty-dependencies.md b/website/Map_Reduce_Folder/developers/thirdparty-dependencies.md index a29033a386..bfacd2f567 100644 --- a/website/Map_Reduce_Folder/developers/thirdparty-dependencies.md +++ b/website/Map_Reduce_Folder/developers/thirdparty-dependencies.md @@ -4,7 +4,7 @@ title: Thirdparty Dependencies redirect_from: - /documentation/developers/thirdparty - /documentation/developers/thirdparty.html - + --- # Adding Thirdparty Dependencies in Maven diff --git a/website/Map_Reduce_Folder/developers/version-control.md b/website/Map_Reduce_Folder/developers/version-control.md index 475a14849b..6f637b270c 100644 --- a/website/Map_Reduce_Folder/developers/version-control.md +++ b/website/Map_Reduce_Folder/developers/version-control.md @@ -5,17 +5,17 @@ redirect_from: - /documentation/developers/version-control - /documentation/developers/version-control.html - + --- # Version control access The Mahout source is mirrored in the **[Apache Mahout GitHub](https://github.com/apache/mahout)** repository. - + ## Web Access (read-only) -The source code can be browsed via the Web at [https://github.com/apache/mahout](https://github.com/apache/mahout). +The source code can be browsed via the Web at [https://github.com/apache/mahout](https://github.com/apache/mahout). ## Anonymous Access (read-only) diff --git a/website/Map_Reduce_Folder/dim-reduction/dimensional-reduction.md b/website/Map_Reduce_Folder/dim-reduction/dimensional-reduction.md index 92fb4d1a22..a260a8cce5 100644 --- a/website/Map_Reduce_Folder/dim-reduction/dimensional-reduction.md +++ b/website/Map_Reduce_Folder/dim-reduction/dimensional-reduction.md @@ -71,7 +71,7 @@ See also: [https://cwiki.apache.org/confluence/display/MAHOUT/SVD+-+Singular+Val The Lanczos algorithm is designed for eigen-decomposition, but like any such algorithm, getting singular vectors out of it is immediate (singular -vectors of matrix A are just the eigenvectors of A^t * A or A * A^t). +vectors of matrix A are just the eigenvectors of A^t * A or A * A^t). Lanczos works by taking a starting seed vector *v* (with cardinality equal to the number of columns of the matrix A), and repeatedly multiplying A by the result: *v'* = A.times(*v*) (and then subtracting off what is @@ -123,41 +123,41 @@ arguments (which can be reproduced by just entering the command with no arguments): - Job-Specific Options: - --input (-i) input Path to job input directory. - --output (-o) output The directory pathname for output. - --numRows (-nr) numRows Number of rows of the input matrix - --numCols (-nc) numCols Number of columns of the input matrix - --rank (-r) rank Desired decomposition rank (note: - only roughly 1/4 to 1/3 of these will - have the top portion of the spectrum) - --symmetric (-sym) symmetric Is the input matrix square and - symmetric? - --cleansvd (-cl) cleansvd Run the EigenVerificationJob to clean - the eigenvectors after SVD - --maxError (-err) maxError Maximum acceptable error - --minEigenvalue (-mev) minEigenvalue Minimum eigenvalue to keep the vector for - --inMemory (-mem) inMemory Buffer eigen matrix into memory (if you have enough!) - --help (-h) Print out help - --tempDir tempDir Intermediate output directory - --startPhase startPhase First phase to run - --endPhase endPhase Last phase to run - - -The short form invocation may be used to perform the SVD on the input data: + Job-Specific Options: + --input (-i) input Path to job input directory. + --output (-o) output The directory pathname for output. + --numRows (-nr) numRows Number of rows of the input matrix + --numCols (-nc) numCols Number of columns of the input matrix + --rank (-r) rank Desired decomposition rank (note: + only roughly 1/4 to 1/3 of these will + have the top portion of the spectrum) + --symmetric (-sym) symmetric Is the input matrix square and + symmetric? + --cleansvd (-cl) cleansvd Run the EigenVerificationJob to clean + the eigenvectors after SVD + --maxError (-err) maxError Maximum acceptable error + --minEigenvalue (-mev) minEigenvalue Minimum eigenvalue to keep the vector for + --inMemory (-mem) inMemory Buffer eigen matrix into memory (if you have enough!) + --help (-h) Print out help + --tempDir tempDir Intermediate output directory + --startPhase startPhase First phase to run + --endPhase endPhase Last phase to run + + +The short form invocation may be used to perform the SVD on the input data: /bin/mahout svd \ - --input (-i) \ - --output (-o) \ - --numRows (-nr) \ + --input (-i) \ + --output (-o) \ + --numRows (-nr) \ --numCols (-nc) \ --rank (-r) \ - --symmetric (-sym) + --symmetric (-sym) The --input argument is the location on HDFS where a SequenceFile (preferably -SequentialAccessSparseVectors instances) lies which you wish to decompose. +SequentialAccessSparseVectors instances) lies which you wish to decompose. Each vector of which has --numcols entries. --numRows is the number of input rows and is used to properly size the matrix data structures. @@ -195,18 +195,18 @@ least 8 bytes/double * rank * numCols), then you will see some speedups on this cleaning process. After execution, the --output directory will have a file named -"cleanEigenvectors" containing the clean eigenvectors. +"cleanEigenvectors" containing the clean eigenvectors. These two steps can also be invoked together by the svd command by using the long form svd invocation: /bin/mahout svd \ - --input (-i) \ - --output (-o) \ - --numRows (-nr) \ + --input (-i) \ + --output (-o) \ + --numRows (-nr) \ --numCols (-nc) \ --rank (-r) \ - --symmetric (-sym) \ + --symmetric (-sym) \ --cleansvd "true" \ --maxError \ --minEigenvalue \ diff --git a/website/Map_Reduce_Folder/dim-reduction/ssvd.md b/website/Map_Reduce_Folder/dim-reduction/ssvd.md index e7c889b6b4..b7d226785c 100644 --- a/website/Map_Reduce_Folder/dim-reduction/ssvd.md +++ b/website/Map_Reduce_Folder/dim-reduction/ssvd.md @@ -8,7 +8,7 @@ redirect_from: # Stochastic Singular Value Decomposition # -Stochastic SVD method in Mahout produces reduced rank Singular Value Decomposition output in its +Stochastic SVD method in Mahout produces reduced rank Singular Value Decomposition output in its strict mathematical definition: ` \(\mathbf{A\approx U}\boldsymbol{\Sigma}\mathbf{V}^{\top}\)`. ##The benefits over other methods are: @@ -23,7 +23,7 @@ strict mathematical definition: ` \(\mathbf{A\approx U}\boldsymbol{\Sigma}\mathb - As of 0.7 trunk, includes PCA and dimensionality reduction workflow (EXPERIMENTAL! Feedback on performance/other PCA related issues/ blogs is greatly appreciated.) -### Map-Reduce characteristics: +### Map-Reduce characteristics: SSVD uses at most 3 MR sequential steps (map-only + map-reduce + 2 optional parallel map-reduce jobs) to produce reduced rank approximation of U, V and S matrices. Additionally, two more map-reduce steps are added for each power iteration step if requested. ##Potential drawbacks: @@ -58,16 +58,16 @@ However, try this R code to simulate a meaningful input: n<-1000 m<-2000 k<-10 - + qi<-1 - + #simulated input svalsim<-diag(k:1) - + usim<- qr.Q(qr(matrix(rnorm(m*k, mean=3), nrow=m,ncol=k))) vsim<- qr.Q(qr( matrix(rnorm(n*k,mean=5), nrow=n,ncol=k))) - - + + x<- usim %*% svalsim %*% t(vsim) @@ -85,8 +85,8 @@ Note: numerical stability of R algorithms may differ from that of Mahout's distr Given an `\(m\times n\)` matrix `\(\mathbf{A}\)`, a target rank `\(k\in\mathbb{N}_{1}\)` -, an oversampling parameter `\(p\in\mathbb{N}_{1}\)`, -and the number of additional power iterations `\(q\in\mathbb{N}_{0}\)`, +, an oversampling parameter `\(p\in\mathbb{N}_{1}\)`, +and the number of additional power iterations `\(q\in\mathbb{N}_{0}\)`, this procedure computes an `\(m\times\left(k+p\right)\)` SVD `\(\mathbf{A\approx U}\boldsymbol{\Sigma}\mathbf{V}^{\top}\)`: @@ -95,34 +95,33 @@ SVD `\(\mathbf{A\approx U}\boldsymbol{\Sigma}\mathbf{V}^{\top}\)`: using Gaussian unit vectors per one of suggestions in [Halko, Martinsson, Tropp]. 2. `\(\mathbf{Y=A\boldsymbol{\Omega}},\,\mathbf{Y}\in\mathbb{R}^{m\times\left(k+p\right)}\)` - + 3. Column-orthonormalize `\(\mathbf{Y}\rightarrow\mathbf{Q}\)` by computing thin decomposition `\(\mathbf{Y}=\mathbf{Q}\mathbf{R}\)`. Also, `\(\mathbf{Q}\in\mathbb{R}^{m\times\left(k+p\right)},\,\mathbf{R}\in\mathbb{R}^{\left(k+p\right)\times\left(k+p\right)}\)`. I denote this as `\(\mathbf{Q}=\mbox{qr}\left(\mathbf{Y}\right).\mathbf{Q}\)` - + 4. `\(\mathbf{B}_{0}=\mathbf{Q}^{\top}\mathbf{A}:\,\,\mathbf{B}\in\mathbb{R}^{\left(k+p\right)\times n}\)`. - + 5. If `\(q>0\)` - repeat: for `\(i=1..q\)`: + repeat: for `\(i=1..q\)`: `\(\mathbf{B}_{i}^{\top}=\mathbf{A}^{\top}\mbox{qr}\left(\mathbf{A}\mathbf{B}_{i-1}^{\top}\right).\mathbf{Q}\)` (power iterations step). 6. Compute Eigensolution of a small Hermitian `\(\mathbf{B}_{q}\mathbf{B}_{q}^{\top}=\mathbf{\hat{U}}\boldsymbol{\Lambda}\mathbf{\hat{U}}^{\top}\)`, `\(\mathbf{B}_{q}\mathbf{B}_{q}^{\top}\in\mathbb{R}^{\left(k+p\right)\times\left(k+p\right)}\)`. - + 7. Singular values `\(\mathbf{\boldsymbol{\Sigma}}=\boldsymbol{\Lambda}^{0.5}\)`, or, in other words, `\(s_{i}=\sqrt{\sigma_{i}}\)`. - + 8. If needed, compute `\(\mathbf{U}=\mathbf{Q}\hat{\mathbf{U}}\)`. - + 9. If needed, compute `\(\mathbf{V}=\mathbf{B}_{q}^{\top}\hat{\mathbf{U}}\boldsymbol{\Sigma}^{-1}\)`. Another way is `\(\mathbf{V}=\mathbf{A}^{\top}\mathbf{U}\boldsymbol{\Sigma}^{-1}\)`. [Halko, Martinsson, Tropp]: http://arxiv.org/abs/0909.4061 - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/clustering/canopy/index.md b/website/Map_Reduce_Folder/docs/latest/algorithms/clustering/canopy/index.md index e88b772eb8..0ed0a279ac 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/clustering/canopy/index.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/clustering/canopy/index.md @@ -4,7 +4,7 @@ title: Canopy Clustering redirect_from: - /docs/latest/algorithms/clustering/canopy/index - /docs/latest/algorithms/clustering/canopy/index.html - + --- ### About @@ -64,7 +64,7 @@ The points are generated as follows: sd=0.1 In the first image, the points are plotted and the 3-sigma boundaries of -their generator are superimposed. +their generator are superimposed. ![sample data](SampleData.png) @@ -123,8 +123,8 @@ thresholds. ### Example val drmA = drmParallelize(dense((1.0, 1.2, 1.3, 1.4), (1.1, 1.5, 2.5, 1.0), (6.0, 5.2, -5.2, 5.3), (7.0,6.0, 5.0, 5.0), (10.0, 1.0, 20.0, -10.0))) - + import org.apache.mahout.math.algorithms.clustering.CanopyClustering - + val model = new CanopyClustering().fit(drmA, 't1 -> 6.5, 't2 -> 5.5, 'distanceMeasure -> 'Chebyshev) model.cluster(drmA).collect diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/clustering/distance-metrics.md b/website/Map_Reduce_Folder/docs/latest/algorithms/clustering/distance-metrics.md index f3f0937941..6ce7480258 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/clustering/distance-metrics.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/clustering/distance-metrics.md @@ -4,7 +4,7 @@ title: Distance Metrics redirect_from: - /docs/latest/algorithms/clustering/distance-metrics - /docs/latest/algorithms/clustering/distance-metrics.html - + --- ### Distance Metrics Supported By Mahout @@ -50,35 +50,35 @@ Weighted Manhattan--> ### Using Distance Metrics -In Mahout one can access the distant metrics directly to measure the distance between two arbitrary vectors, or -can specify which distance metric to use as part of an algorithm. In the latter case the distance metric is called -by `Symbol`, we never pass Distance metrics directly to an algorithm. This design choice, in part has to do with -serialization of object and keeping the engine bindings as simple as possible. Behind the scenes, the only thing +In Mahout one can access the distant metrics directly to measure the distance between two arbitrary vectors, or +can specify which distance metric to use as part of an algorithm. In the latter case the distance metric is called +by `Symbol`, we never pass Distance metrics directly to an algorithm. This design choice, in part has to do with +serialization of object and keeping the engine bindings as simple as possible. Behind the scenes, the only thing that is serialized and sent to the workers is a number which specifies what distant metric to use- this is much more abstract and easier to maintain on the back end than making sure each function can be serialized by any arbitrary engine. -We feel from the user perspective, it may seem quirky but causes no decrease in usability. If a user wishes to use a +We feel from the user perspective, it may seem quirky but causes no decrease in usability. If a user wishes to use a custom distance metric- simply add it to [math-scala/src/main/org/apache/mahout/math/common/DistanceMetrics.scala](https://github.com/apache/mahout/blob/trunk/core/src/main/scala/org/apache/mahout/math/algorithms/common/distance/DistanceMetrics.scala) -and recompile. +and recompile. ### Examples **Meausring the distance between two vectors** import org.apache.mahout.math.algorithms.common.distance._ - + val v1 = dvec(1.0, 1.5, -1.2, 3.5) val v2 = dvec(0.1, -1.4, 10.5, 3.2) - + Cosine.distance(v1, v2) **Using distance in clustering** import org.apache.mahout.math.algorithms.clustering.CanopyClustering - val drmA = drmParallelize(dense((1.0, 1.2, 1.3, 1.4), - (1.1, 1.5, 2.5, 1.0), - (6.0, 5.2, -5.2, 5.3), - (7.0,6.0, 5.0, 5.0), + val drmA = drmParallelize(dense((1.0, 1.2, 1.3, 1.4), + (1.1, 1.5, 2.5, 1.0), + (6.0, 5.2, -5.2, 5.3), + (7.0,6.0, 5.0, 5.0), (10.0, 1.0, 20.0, -10.0))) - + val model = new CanopyClustering().fit(drmA, 'distanceMeasure -> 'Cosine) diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/linear-algebra/d-qr.md b/website/Map_Reduce_Folder/docs/latest/algorithms/linear-algebra/d-qr.md index 8f947a6104..ed1cbc1a6c 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/linear-algebra/d-qr.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/linear-algebra/d-qr.md @@ -3,24 +3,24 @@ layout: doc-page title: Distributed Cholesky QR redirect_from: - /docs/latest/algorithms/linear-algebra/d-qr - - /docs/latest/algorithms/linear-algebra/d-qr.html + - /docs/latest/algorithms/linear-algebra/d-qr.html --- ## Intro Mahout has a distributed implementation of QR decomposition for tall thin matrices[1]. -## Algorithm +## Algorithm -For the classic QR decomposition of the form $$ \mathbf{A}=\mathbf{QR},\mathbf{A}\in\mathbb{R}^{m\times n} $$ -a distributed version is fairly easily achieved if $$ \mathbf{A} $$ is tall and thin such that -$$ \mathbf{A}^{\top}\mathbf{A} $$ fits in memory, i.e. *m* is large but *n* < ~5000 Under such circumstances, -only $$ \mathbf{A} $$ and $$ \mathbf{Q} $$ are distributed matrices and $$ \mathbf{A^{\top}A} $$ and -$$ \mathbf{R} $$ are in-core products. We just compute the in-core version of the Cholesky decomposition +For the classic QR decomposition of the form $$ \mathbf{A}=\mathbf{QR},\mathbf{A}\in\mathbb{R}^{m\times n} $$ +a distributed version is fairly easily achieved if $$ \mathbf{A} $$ is tall and thin such that +$$ \mathbf{A}^{\top}\mathbf{A} $$ fits in memory, i.e. *m* is large but *n* < ~5000 Under such circumstances, +only $$ \mathbf{A} $$ and $$ \mathbf{Q} $$ are distributed matrices and $$ \mathbf{A^{\top}A} $$ and +$$ \mathbf{R} $$ are in-core products. We just compute the in-core version of the Cholesky decomposition in the form of $$ \mathbf{LL}^{\top}= \mathbf{A}^{\top}\mathbf{A}$$. After that we take $$ \mathbf{R}= \mathbf{L}^{\top} $$ - and $$ \mathbf{Q}=\mathbf{A}\left(\mathbf{L}^{\top}\right)^{-1} $$. The latter is easily achieved by multiplying each - vertical block of $$ \mathbf{A} $$ by $$ \left(\mathbf{L}^{\top}\right)^{-1} $$. (There is no actual matrix inversion - happening). + and $$ \mathbf{Q}=\mathbf{A}\left(\mathbf{L}^{\top}\right)^{-1} $$. The latter is easily achieved by multiplying each + vertical block of $$ \mathbf{A} $$ by $$ \left(\mathbf{L}^{\top}\right)^{-1} $$. (There is no actual matrix inversion + happening). @@ -28,7 +28,7 @@ in the form of $$ \mathbf{LL}^{\top}= \mathbf{A}^{\top}\mathbf{A}$$. After that Mahout `dqrThin(...)` is implemented in the mahout `math-scala` algebraic optimizer which translates Mahout's R-like linear algebra operators into a physical plan for both Spark and H2O distributed engines. - def dqrThin[K: ClassTag](A: DrmLike[K], checkRankDeficiency: Boolean = true): (DrmLike[K], Matrix) = { + def dqrThin[K: ClassTag](A: DrmLike[K], checkRankDeficiency: Boolean = true): (DrmLike[K], Matrix) = { if (drmA.ncol > 5000) log.warn("A is too fat. A'A must fit in memory and easily broadcasted.") implicit val ctx = drmA.context @@ -53,13 +53,12 @@ The scala `dqrThin(...)` method can easily be called in any Spark or H2O applica import org.apache.mahout.math._ import decompositions._ import drm._ - + val(drmQ, inCoreR) = dqrThin(drma) - + ## References [1]: [Mahout Scala and Mahout Spark Bindings for Linear Algebra Subroutines](http://mahout.apache.org/users/sparkbindings/ScalaSparkBindings.pdf) [2]: [Mahout Spark and Scala Bindings](http://mahout.apache.org/users/sparkbindings/home.html) - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/linear-algebra/d-spca.md b/website/Map_Reduce_Folder/docs/latest/algorithms/linear-algebra/d-spca.md index 88ed2928fd..d4210ba9df 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/linear-algebra/d-spca.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/linear-algebra/d-spca.md @@ -1,6 +1,6 @@ --- layout: doc-page -title: Distributed Stochastic PCA +title: Distributed Stochastic PCA redirect_from: - /docs/latest/algorithms/linear-algebra/d-spca - /docs/latest/algorithms/linear-algebra/d-spca.html @@ -40,7 +40,7 @@ Given an *m* `\(\times\)` *n* matrix `\(\mathbf{A}\)`, a target rank *k*, and an Mahout `dspca(...)` is implemented in the mahout `math-scala` algebraic optimizer which translates Mahout's R-like linear algebra operators into a physical plan for both Spark and H2O distributed engines. - def dspca[K](drmA: DrmLike[K], k: Int, p: Int = 15, q: Int = 0): + def dspca[K](drmA: DrmLike[K], k: Int, p: Int = 15, q: Int = 0): (DrmLike[K], DrmLike[Int], Vector) = { // Some mapBlock() calls need it @@ -165,11 +165,11 @@ The scala `dspca(...)` method can easily be called in any Spark, Flink, or H2O a import org.apache.mahout.math._ import decompositions._ import drm._ - + val (drmU, drmV, s) = dspca(drmA, k=200, q=1) Note the parameter is optional and its default value is zero. - + ## References [1]: Lyubimov and Palumbo, ["Apache Mahout: Beyond MapReduce; Distributed Algorithm Design"](https://www.amazon.com/Apache-Mahout-MapReduce-Dmitriy-Lyubimov/dp/1523775785) diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/linear-algebra/d-ssvd.md b/website/Map_Reduce_Folder/docs/latest/algorithms/linear-algebra/d-ssvd.md index cd936aa005..6e2ffc5d71 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/linear-algebra/d-ssvd.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/linear-algebra/d-ssvd.md @@ -1,6 +1,6 @@ --- layout: doc-page -title: Distributed Stochastic Singular Value Decomposition +title: Distributed Stochastic Singular Value Decomposition redirect_from: - /docs/latest/algorithms/linear-algebra/d-ssvd - /docs/latest/algorithms/linear-algebra/d-ssvd.html @@ -14,8 +14,8 @@ Mahout has a distributed implementation of Stochastic Singular Value Decompositi Given an \(m\times n\) matrix \(\mathbf{A}\), a target rank \(k\in\mathbb{N}_{1}\) -, an oversampling parameter \(p\in\mathbb{N}_{1}\), -and the number of additional power iterations \(q\in\mathbb{N}_{0}\), +, an oversampling parameter \(p\in\mathbb{N}_{1}\), +and the number of additional power iterations \(q\in\mathbb{N}_{0}\), this procedure computes an \(m\times\left(k+p\right)\) SVD \(\mathbf{A\approx U}\boldsymbol{\Sigma}\mathbf{V}^{\top}\): @@ -24,24 +24,24 @@ SVD \(\mathbf{A\approx U}\boldsymbol{\Sigma}\mathbf{V}^{\top}\): using Gaussian unit vectors per one of suggestions in [Halko, Martinsson, Tropp]. 2. \(\mathbf{Y=A\boldsymbol{\Omega}},\,\mathbf{Y}\in\mathbb{R}^{m\times\left(k+p\right)}\) - + 3. Column-orthonormalize \(\mathbf{Y}\rightarrow\mathbf{Q}\) by computing thin decomposition \(\mathbf{Y}=\mathbf{Q}\mathbf{R}\). Also, \(\mathbf{Q}\in\mathbb{R}^{m\times\left(k+p\right)},\,\mathbf{R}\in\mathbb{R}^{\left(k+p\right)\times\left(k+p\right)}\); denoted as \(\mathbf{Q}=\mbox{qr}\left(\mathbf{Y}\right).\mathbf{Q}\) 4. \(\mathbf{B}_{0}=\mathbf{Q}^{\top}\mathbf{A}:\,\,\mathbf{B}\in\mathbb{R}^{\left(k+p\right)\times n}\). - + 5. If \(q>0\) - repeat: for \(i=1..q\): + repeat: for \(i=1..q\): \(\mathbf{B}_{i}^{\top}=\mathbf{A}^{\top}\mbox{qr}\left(\mathbf{A}\mathbf{B}_{i-1}^{\top}\right).\mathbf{Q}\) (power iterations step). 6. Compute Eigensolution of a small Hermitian \(\mathbf{B}_{q}\mathbf{B}_{q}^{\top}=\mathbf{\hat{U}}\boldsymbol{\Lambda}\mathbf{\hat{U}}^{\top}\), \(\mathbf{B}_{q}\mathbf{B}_{q}^{\top}\in\mathbb{R}^{\left(k+p\right)\times\left(k+p\right)}\). - + 7. Singular values \(\mathbf{\boldsymbol{\Sigma}}=\boldsymbol{\Lambda}^{0.5}\), or, in other words, \(s_{i}=\sqrt{\sigma_{i}}\). - + 8. If needed, compute \(\mathbf{U}=\mathbf{Q}\hat{\mathbf{U}}\). 9. If needed, compute \(\mathbf{V}=\mathbf{B}_{q}^{\top}\hat{\mathbf{U}}\boldsymbol{\Sigma}^{-1}\). @@ -70,7 +70,7 @@ Mahout `dssvd(...)` is implemented in the mahout `math-scala` algebraic optimize // We represent Omega by its seed. val omegaSeed = RandomUtils.getRandom().nextInt() - // Compute Y = A*Omega. + // Compute Y = A*Omega. var drmY = drmAcp.mapBlock(ncol = r) { case (keys, blockA) => val blockY = blockA %*% Matrices.symmetricUniformView(n, r, omegaSeed) @@ -83,19 +83,19 @@ Mahout `dssvd(...)` is implemented in the mahout `math-scala` algebraic optimize if (q == 0) drmQ = drmQ.checkpoint() var drmBt = drmAcp.t %*% drmQ - + // Checkpoint B' if last iteration if (q == 0) drmBt = drmBt.checkpoint() for (i <- 0 until q) { drmY = drmAcp %*% drmBt - drmQ = dqrThin(drmY.checkpoint())._1 - + drmQ = dqrThin(drmY.checkpoint())._1 + // Checkpoint Q if last iteration if (i == q - 1) drmQ = drmQ.checkpoint() - + drmBt = drmAcp.t %*% drmQ - + // Checkpoint B' if last iteration if (i == q - 1) drmBt = drmBt.checkpoint() } @@ -104,7 +104,7 @@ Mahout `dssvd(...)` is implemented in the mahout `math-scala` algebraic optimize val s = d.sqrt // Since neither drmU nor drmV are actually computed until actually used - // we don't need the flags instructing compute (or not compute) either of the U,V outputs + // we don't need the flags instructing compute (or not compute) either of the U,V outputs val drmU = drmQ %*% inCoreUHat val drmV = drmBt %*% (inCoreUHat %*%: diagv(1 /: s)) @@ -121,11 +121,11 @@ The scala `dssvd(...)` method can easily be called in any Spark or H2O applicati import org.apache.mahout.math._ import decompositions._ import drm._ - - + + val(drmU, drmV, s) = dssvd(drma, k = 40, q = 1) - + ## References [1]: [Mahout Scala and Mahout Spark Bindings for Linear Algebra Subroutines](http://mahout.apache.org/users/sparkbindings/ScalaSparkBindings.pdf) @@ -136,6 +136,3 @@ approximations of matrices](http://amath.colorado.edu/faculty/martinss/Pubs/2012 [2]: [Halko, Martinsson, Tropp](http://arxiv.org/abs/0909.4061) [3]: [Mahout Spark and Scala Bindings](http://mahout.apache.org/users/sparkbindings/home.html) - - - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/linear-algebra/index.md b/website/Map_Reduce_Folder/docs/latest/algorithms/linear-algebra/index.md index a7ea92e321..e1f45ba56c 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/linear-algebra/index.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/linear-algebra/index.md @@ -7,10 +7,9 @@ redirect_from: --- Mahout has a number of distributed linear algebra "algorithms" that, in concert with the mathematically expressive R-Like Scala DSL, make it possible for users to quickly "roll their own" distributed algorithms. - + [Distributed QR Decomposition](d-qr.html) [Distributed Stochastic Principal Component Analysis](d-spca.html) [Distributed Stochastic Singular Value Decomposition](d-ssvd.html) - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/bayesian.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/bayesian.md index 364fc2cd83..883fa676b8 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/bayesian.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/bayesian.md @@ -1,6 +1,6 @@ --- layout: doc-page -title: (Deprecated) +title: (Deprecated) redirect_from: - /docs/latest/algorithms/map-reduce/classification/bayesian - /docs/latest/algorithms/map-reduce/classification/bayesian.html @@ -13,7 +13,7 @@ redirect_from: Mahout currently has two Naive Bayes Map-Reduce implementations. The first is standard Multinomial Naive Bayes. The second is an implementation of Transformed Weight-normalized Complement Naive Bayes as introduced by Rennie et al. [[1]](http://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf). We refer to the former as Bayes and the latter as CBayes. -Where Bayes has long been a standard in text classification, CBayes is an extension of Bayes that performs particularly well on datasets with skewed classes and has been shown to be competitive with algorithms of higher complexity such as Support Vector Machines. +Where Bayes has long been a standard in text classification, CBayes is an extension of Bayes that performs particularly well on datasets with skewed classes and has been shown to be competitive with algorithms of higher complexity such as Support Vector Machines. ## Implementations @@ -21,15 +21,15 @@ Both Bayes and CBayes are currently trained via MapReduce Jobs. Testing and clas ## Preprocessing and Algorithm -As described in [[1]](http://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf) Mahout Naive Bayes is broken down into the following steps (assignments are over all possible index values): +As described in [[1]](http://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf) Mahout Naive Bayes is broken down into the following steps (assignments are over all possible index values): - Let `\(\vec{d}=(\vec{d_1},...,\vec{d_n})\)` be a set of documents; `\(d_{ij}\)` is the count of word `\(i\)` in document `\(j\)`. - Let `\(\vec{y}=(y_1,...,y_n)\)` be their labels. -- Let `\(\alpha_i\)` be a smoothing parameter for all words in the vocabulary; let `\(\alpha=\sum_i{\alpha_i}\)`. +- Let `\(\alpha_i\)` be a smoothing parameter for all words in the vocabulary; let `\(\alpha=\sum_i{\alpha_i}\)`. - **Preprocessing**(via seq2Sparse) TF-IDF transformation and L2 length normalization of `\(\vec{d}\)` - 1. `\(d_{ij} = \sqrt{d_{ij}}\)` - 2. `\(d_{ij} = d_{ij}\left(\log{\frac{\sum_k1}{\sum_k\delta_{ik}+1}}+1\right)\)` - 3. `\(d_{ij} =\frac{d_{ij}}{\sqrt{\sum_k{d_{kj}^2}}}\)` + 1. `\(d_{ij} = \sqrt{d_{ij}}\)` + 2. `\(d_{ij} = d_{ij}\left(\log{\frac{\sum_k1}{\sum_k\delta_{ik}+1}}+1\right)\)` + 3. `\(d_{ij} =\frac{d_{ij}}{\sqrt{\sum_k{d_{kj}^2}}}\)` - **Training: Bayes**`\((\vec{d},\vec{y})\)` calculate term weights `\(w_{ci}\)` as: 1. `\(\hat\theta_{ci}=\frac{d_{ic}+\alpha_i}{\sum_k{d_{kc}}+\alpha}\)` 2. `\(w_{ci}=\log{\hat\theta_{ci}}\)` @@ -41,19 +41,19 @@ As described in [[1]](http://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf) 1. Let `\(\vec{t}= (t_1,...,t_n)\)` be a test document; let `\(t_i\)` be the count of the word `\(t\)`. 2. Label the document according to `\(l(t)=\arg\max_c \sum\limits_{i} t_i w_{ci}\)` -As we can see, the main difference between Bayes and CBayes is the weight calculation step. Where Bayes weighs terms more heavily based on the likelihood that they belong to class `\(c\)`, CBayes seeks to maximize term weights on the likelihood that they do not belong to any other class. +As we can see, the main difference between Bayes and CBayes is the weight calculation step. Where Bayes weighs terms more heavily based on the likelihood that they belong to class `\(c\)`, CBayes seeks to maximize term weights on the likelihood that they do not belong to any other class. ## Running from the command line -Mahout provides CLI drivers for all above steps. Here we will give a simple overview of Mahout CLI commands used to preprocess the data, train the model and assign labels to the training set. An [example script](https://github.com/apache/mahout/blob/master/examples/bin/classify-20newsgroups.sh) is given for the full process from data acquisition through classification of the classic [20 Newsgroups corpus](https://mahout.apache.org/users/classification/twenty-newsgroups.html). +Mahout provides CLI drivers for all above steps. Here we will give a simple overview of Mahout CLI commands used to preprocess the data, train the model and assign labels to the training set. An [example script](https://github.com/apache/mahout/blob/master/examples/bin/classify-20newsgroups.sh) is given for the full process from data acquisition through classification of the classic [20 Newsgroups corpus](https://mahout.apache.org/users/classification/twenty-newsgroups.html). - **Preprocessing:** For a set of Sequence File Formatted documents in PATH_TO_SEQUENCE_FILES the [mahout seq2sparse](https://mahout.apache.org/users/basics/creating-vectors-from-text.html) command performs the TF-IDF transformations (-wt tfidf option) and L2 length normalization (-n 2 option) as follows: - mahout seq2sparse - -i ${PATH_TO_SEQUENCE_FILES} - -o ${PATH_TO_TFIDF_VECTORS} - -nv + mahout seq2sparse + -i ${PATH_TO_SEQUENCE_FILES} + -o ${PATH_TO_TFIDF_VECTORS} + -nv -n 2 -wt tfidf @@ -61,78 +61,78 @@ For a set of Sequence File Formatted documents in PATH_TO_SEQUENCE_FILES the [ma The model is then trained using `mahout trainnb` . The default is to train a Bayes model. The -c option is given to train a CBayes model: mahout trainnb - -i ${PATH_TO_TFIDF_VECTORS} - -o ${PATH_TO_MODEL}/model - -li ${PATH_TO_MODEL}/labelindex - -ow + -i ${PATH_TO_TFIDF_VECTORS} + -o ${PATH_TO_MODEL}/model + -li ${PATH_TO_MODEL}/labelindex + -ow -c - **Label Assignment/Testing:** Classification and testing on a holdout set can then be performed via `mahout testnb`. Again, the -c option indicates that the model is CBayes. The -seq option tells `mahout testnb` to run sequentially: - mahout testnb + mahout testnb -i ${PATH_TO_TFIDF_TEST_VECTORS} - -m ${PATH_TO_MODEL}/model - -l ${PATH_TO_MODEL}/labelindex - -ow - -o ${PATH_TO_OUTPUT} - -c + -m ${PATH_TO_MODEL}/model + -l ${PATH_TO_MODEL}/labelindex + -ow + -o ${PATH_TO_OUTPUT} + -c -seq ## Command line options - **Preprocessing:** - + Only relevant parameters used for Bayes/CBayes as detailed above are shown. Several other transformations can be performed by `mahout seq2sparse` and used as input to Bayes/CBayes. For a full list of `mahout seq2Sparse` options see the [Creating vectors from text](https://mahout.apache.org/users/basics/creating-vectors-from-text.html) page. - mahout seq2sparse - --output (-o) output The directory pathname for output. - --input (-i) input Path to job input directory. - --weight (-wt) weight The kind of weight to use. Currently TF - or TFIDF. Default: TFIDF - --norm (-n) norm The norm to use, expressed as either a - float or "INF" if you want to use the - Infinite norm. Must be greater or equal - to 0. The default is not to normalize - --overwrite (-ow) If set, overwrite the output directory - --sequentialAccessVector (-seq) (Optional) Whether output vectors should - be SequentialAccessVectors. If set true - else false - --namedVector (-nv) (Optional) Whether output vectors should - be NamedVectors. If set true else false + mahout seq2sparse + --output (-o) output The directory pathname for output. + --input (-i) input Path to job input directory. + --weight (-wt) weight The kind of weight to use. Currently TF + or TFIDF. Default: TFIDF + --norm (-n) norm The norm to use, expressed as either a + float or "INF" if you want to use the + Infinite norm. Must be greater or equal + to 0. The default is not to normalize + --overwrite (-ow) If set, overwrite the output directory + --sequentialAccessVector (-seq) (Optional) Whether output vectors should + be SequentialAccessVectors. If set true + else false + --namedVector (-nv) (Optional) Whether output vectors should + be NamedVectors. If set true else false - **Training:** mahout trainnb - --input (-i) input Path to job input directory. - --output (-o) output The directory pathname for output. + --input (-i) input Path to job input directory. + --output (-o) output The directory pathname for output. --alphaI (-a) alphaI Smoothing parameter. Default is 1.0 - --trainComplementary (-c) Train complementary? Default is false. - --labelIndex (-li) labelIndex The path to store the label index in - --overwrite (-ow) If present, overwrite the output directory - before running job - --help (-h) Print out help - --tempDir tempDir Intermediate output directory - --startPhase startPhase First phase to run + --trainComplementary (-c) Train complementary? Default is false. + --labelIndex (-li) labelIndex The path to store the label index in + --overwrite (-ow) If present, overwrite the output directory + before running job + --help (-h) Print out help + --tempDir tempDir Intermediate output directory + --startPhase startPhase First phase to run --endPhase endPhase Last phase to run - **Testing:** - mahout testnb - --input (-i) input Path to job input directory. - --output (-o) output The directory pathname for output. - --overwrite (-ow) If present, overwrite the output directory - before running job + mahout testnb + --input (-i) input Path to job input directory. + --output (-o) output The directory pathname for output. + --overwrite (-ow) If present, overwrite the output directory + before running job - - --model (-m) model The path to the model built during training - --testComplementary (-c) Test complementary? Default is false. - --runSequential (-seq) Run sequential? - --labelIndex (-l) labelIndex The path to the location of the label index - --help (-h) Print out help - --tempDir tempDir Intermediate output directory - --startPhase startPhase First phase to run - --endPhase endPhase Last phase to run + + --model (-m) model The path to the model built during training + --testComplementary (-c) Test complementary? Default is false. + --runSequential (-seq) Run sequential? + --labelIndex (-l) labelIndex The path to the location of the label index + --help (-h) Print out help + --tempDir tempDir Intermediate output directory + --startPhase startPhase First phase to run + --endPhase endPhase Last phase to run ## Examples @@ -140,9 +140,7 @@ Classification and testing on a holdout set can then be performed via `mahout te Mahout provides an example for Naive Bayes classification: 1. [Classify 20 Newsgroups](twenty-newsgroups.html) - + ## References [1]: Jason D. M. Rennie, Lawerence Shih, Jamie Teevan, David Karger (2003). [Tackling the Poor Assumptions of Naive Bayes Text Classifiers](http://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf). Proceedings of the Twentieth International Conference on Machine Learning (ICML-2003). - - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/class-discovery.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/class-discovery.md index 28a1675d32..905d84b234 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/class-discovery.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/class-discovery.md @@ -11,7 +11,7 @@ redirect_from: See http://www.cs.bham.ac.uk/~wbl/biblio/gecco1999/GP-417.pdf CDGA uses a Genetic Algorithm to discover a classification rule for a given -dataset. +dataset. A dataset can be seen as a table: @@ -114,15 +114,15 @@ file * param 7 : mutation precision * param 8 : population size * param 9 : number of generations before the program stops - + For more information about 4th parameter, please see [Multi-point Crossover|http://www.geatbx.com/docu/algindex-03.html#P616_36571] . For a detailed explanation about the 5th, 6th and 7th parameters, please see [Real Valued Mutation|http://www.geatbx.com/docu/algindex-04.html#P659_42386] . - + *TODO*: Fill in where to find the output and what it means. - + h1. The info file: To run properly, CDGA needs some informations about the dataset. Each dataset should be accompanied by an .infos file that contains the needed @@ -136,10 +136,10 @@ describes it, it can be one of the following: if the attribute is categorial (nominal), and its possible values * NUMERICAL, min, max if the attribute is numerical, and its min and max values - + This file can be generated automaticaly using a special tool available with CDGA. - + * the tool searches for an existing infos file (*must be filled by the @@ -149,7 +149,7 @@ the ".infos" extension, that contain the type of the attributes: ** 'C' categorical attribute ** 'L' label (this also a categorical attribute) ** 'I' to ignore the attribute - each attribute is in a separate + each attribute is in a separate * A Hadoop job is used to parse the dataset and collect the informations. This means that *the dataset can be distributed over HDFS*. * the results are written back in the same .info file, with the correct diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/collocations.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/collocations.md index 8a2d76ed99..c1b5d8eeeb 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/collocations.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/collocations.md @@ -76,7 +76,7 @@ The tools that the collocation identification algorithm are embeeded within either consume tokenized text as input or provide the ability to specify an implementation of the Lucene Analyzer class perform tokenization in order to form ngrams. The tokens are passed through a Lucene ShingleFilter to -produce NGrams of the desired length. +produce NGrams of the desired length. Given the text "Alice was beginning to get very tired" as an example, Lucene's StandardAnalyzer produces the tokens 'alice', 'beginning', 'get', @@ -103,20 +103,20 @@ defaults to 2 or is set to a number of 2 or greater. The --minLLR option can be used to control the cutoff that prevents collocations below the specified LLR score from being emitted, and the --minSupport argument can be used to filter out collocations that appear below a certain number of -times. +times. bin/mahout seq2sparse - - Usage: + + Usage: [--minSupport --analyzerName --chunkSize --output --input --minDF --maxDFPercent --weight --norm --minLLR - --numReducers --maxNGramSize --overwrite --help + --numReducers --maxNGramSize --overwrite --help --sequentialAccessVector] - Options + Options - --minSupport (-s) minSupport (Optional) Minimum Support. Default Value: 2 + --minSupport (-s) minSupport (Optional) Minimum Support. Default Value: 2 --analyzerName (-a) analyzerName The class name of the analyzer @@ -128,33 +128,33 @@ times. --minDF (-md) minDF The minimum document frequency. Default is 1 - --maxDFPercent (-x) maxDFPercent The max percentage of docs for the DF. Can be used to remove + --maxDFPercent (-x) maxDFPercent The max percentage of docs for the DF. Can be used to remove really high frequency terms. Expressed as an - integer between 0 and 100. Default is 99. + integer between 0 and 100. Default is 99. - --weight (-wt) weight The kind of weight to use. Currently TF - or TFIDF + --weight (-wt) weight The kind of weight to use. Currently TF + or TFIDF - --norm (-n) norm The norm to use, expressed as either a - float or "INF" if you want to use the - Infinite norm. Must be greater orequal - to 0. The default is not to normalize + --norm (-n) norm The norm to use, expressed as either a + float or "INF" if you want to use the + Infinite norm. Must be greater orequal + to 0. The default is not to normalize - --minLLR (-ml) minLLR (Optional)The minimum Log Likelihood + --minLLR (-ml) minLLR (Optional)The minimum Log Likelihood Ratio(Float) Default is 1.0 - - --numReducers (-nr) numReducers (Optional) Number of reduce tasks. - Default Value: 1 - --maxNGramSize (-ng) ngramSize (Optional) The maximum size of ngrams to - create (2 = bigrams, 3 = trigrams, etc) - Default Value:2 - - --overwrite (-w) If set, overwrite the output directory - --help (-h) Print out help - --sequentialAccessVector (-seq) (Optional) Whether output vectors should - be SequentialAccessVectors If set true - else false + --numReducers (-nr) numReducers (Optional) Number of reduce tasks. + Default Value: 1 + + --maxNGramSize (-ng) ngramSize (Optional) The maximum size of ngrams to + create (2 = bigrams, 3 = trigrams, etc) + Default Value:2 + + --overwrite (-w) If set, overwrite the output directory + --help (-h) Print out help + --sequentialAccessVector (-seq) (Optional) Whether output vectors should + be SequentialAccessVectors If set true + else false @@ -162,44 +162,44 @@ times. bin/mahout org.apache.mahout.vectorizer.collocations.llr.CollocDriver - - Usage: - [--input --output --maxNGramSize --overwrite - --minSupport --minLLR --numReducers + + Usage: + [--input --output --maxNGramSize --overwrite + --minSupport --minLLR --numReducers --analyzerName --preprocess --unigram --help] - Options + Options + + --input (-i) input The Path for input files. - --input (-i) input The Path for input files. + --output (-o) output The Path write output to - --output (-o) output The Path write output to + --maxNGramSize (-ng) ngramSize (Optional) The maximum size of ngramsto + create (2 = bigrams, 3 = trigrams,etc) + Default Value:2 - --maxNGramSize (-ng) ngramSize (Optional) The maximum size of ngramsto - create (2 = bigrams, 3 = trigrams,etc) - Default Value:2 - - --overwrite (-w) If set, overwrite the outputdirectory + --overwrite (-w) If set, overwrite the outputdirectory - --minSupport (-s) minSupport (Optional) Minimum Support. Default - Value: 2 + --minSupport (-s) minSupport (Optional) Minimum Support. Default + Value: 2 --minLLR (-ml) minLLR (Optional)The minimum Log Likelihood - Ratio(Float) Default is 1.0 - - --numReducers (-nr) numReducers (Optional) Number of reduce tasks. - Default Value: 1 - - --analyzerName (-a) analyzerName The class name of the analyzer - - --preprocess (-p) If set, input is SequenceFile - where the value is the document, which - will be tokenized using the specified - analyzer. - - --unigram (-u) If set, unigrams will be emitted inthe + Ratio(Float) Default is 1.0 + + --numReducers (-nr) numReducers (Optional) Number of reduce tasks. + Default Value: 1 + + --analyzerName (-a) analyzerName The class name of the analyzer + + --preprocess (-p) If set, input is SequenceFile + where the value is the document, which + will be tokenized using the specified + analyzer. + + --unigram (-u) If set, unigrams will be emitted inthe final output alongside collocations - - --help (-h) Print out help + + --help (-h) Print out help @@ -222,7 +222,7 @@ Generates NGrams and counts frequencies for ngrams, head and tail subgrams. #### Map: CollocMapper -Input k: Text (documentId), v: StringTuple (tokens) +Input k: Text (documentId), v: StringTuple (tokens) Each call to the mapper passes in the full set of tokens for the corresponding document using a StringTuple. The ShingleFilter is run across @@ -235,7 +235,7 @@ Once this is done, ngrams are split into head and tail portions. A key of type G head_key(EMPTY) -> (head subgram, head frequency) - head_key(ngram) -> (ngram, ngram frequency) + head_key(ngram) -> (ngram, ngram frequency) tail_key(EMPTY) -> (tail subgram, tail frequency) @@ -285,7 +285,7 @@ The input will be traversed in the following order: (head subgram, frequency 1) (head subgram, frequency 2) - ... + ... (head subgram, frequency N) (ngram 1, frequency 1) (ngram 1, frequency 2) @@ -383,4 +383,3 @@ By default in seq2sparse, or if the -u option is provided to the CollocDriver, unigrams (single tokens) will be passed through the job and each token's frequency will be calculated. As with ngrams, unigrams are subject to filtering with minSupport and minLLR. - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/hidden-markov-models.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/hidden-markov-models.md index e47d2b8a18..8cbf28f00d 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/hidden-markov-models.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/hidden-markov-models.md @@ -13,7 +13,7 @@ redirect_from: Hidden Markov Models are used in multiple areas of Machine Learning, such as speech recognition, handwritten letter recognition or natural language -processing. +processing. ## Formal Definition @@ -69,19 +69,19 @@ Now run the baumwelch job to train your model, after first setting MAHOUT_LOCAL Output like the following should appear in the console. - Initial probabilities: - 0 1 2 - 1.0 0.0 3.5659361683006626E-251 + Initial probabilities: + 0 1 2 + 1.0 0.0 3.5659361683006626E-251 Transition matrix: - 0 1 2 - 0 6.098919959130616E-5 0.9997275322964165 2.1147850399214744E-4 - 1 7.404648706054873E-37 0.9086408633885092 0.09135913661149081 - 2 0.2284374545687356 7.01786289571088E-11 0.7715625453610858 - Emission matrix: - 0 1 2 3 - 0 0.9999997858591223 2.0536163836449762E-39 2.1414087769942127E-7 1.052441093535389E-27 - 1 7.495656581383351E-34 0.2241269055449904 0.4510889999455847 0.32478409450942497 - 2 0.815051477991782 0.18494852200821799 8.465660634827592E-33 2.8603899591778015E-36 + 0 1 2 + 0 6.098919959130616E-5 0.9997275322964165 2.1147850399214744E-4 + 1 7.404648706054873E-37 0.9086408633885092 0.09135913661149081 + 2 0.2284374545687356 7.01786289571088E-11 0.7715625453610858 + Emission matrix: + 0 1 2 3 + 0 0.9999997858591223 2.0536163836449762E-39 2.1414087769942127E-7 1.052441093535389E-27 + 1 7.495656581383351E-34 0.2241269055449904 0.4510889999455847 0.32478409450942497 + 2 0.815051477991782 0.18494852200821799 8.465660634827592E-33 2.8603899591778015E-36 14/03/22 09:52:21 INFO driver.MahoutDriver: Program took 180 ms (Minutes: 0.003) The model trained with the input set now is in the file 'hmm-model', which we can use to build a predicted sequence. @@ -90,7 +90,7 @@ The model trained with the input set now is in the file 'hmm-model', which we ca To see the predictions: - $ cat hmm-predictions + $ cat hmm-predictions 0 1 3 3 2 2 2 2 1 2 diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/logistic-regression.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/logistic-regression.md index d64527bb37..06399e2b4b 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/logistic-regression.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/logistic-regression.md @@ -14,7 +14,7 @@ occurrence of an event. It makes use of several predictor variables that may be either numerical or categories. Logistic regression is the standard industry workhorse that underlies many -production fraud detection and advertising quality and targeting products. +production fraud detection and advertising quality and targeting products. The Mahout implementation uses Stochastic Gradient Descent (SGD) to all large training sets to be used. @@ -23,18 +23,18 @@ Paul Komarek](http://repository.cmu.edu/cgi/viewcontent.cgi?article=1221&context See MAHOUT-228 for the main JIRA issue for SGD. -A more detailed overview of the Mahout Linear Regression classifier and [detailed discription of building a Logistic Regression classifier](http://blog.trifork.com/2014/02/04/an-introduction-to-mahouts-logistic-regression-sgd-classifier/) for the classic [Iris flower dataset](http://en.wikipedia.org/wiki/Iris_flower_data_set) is also available [2]. +A more detailed overview of the Mahout Linear Regression classifier and [detailed discription of building a Logistic Regression classifier](http://blog.trifork.com/2014/02/04/an-introduction-to-mahouts-logistic-regression-sgd-classifier/) for the classic [Iris flower dataset](http://en.wikipedia.org/wiki/Iris_flower_data_set) is also available [2]. An example of training a Logistic Regression classifier for the [UCI Bank Marketing Dataset](http://mlr.cs.umass.edu/ml/datasets/Bank+Marketing) can be found [on the Mahout website](http://mahout.apache.org/users/classification/bankmarketing-example.html) [3]. -An example of training and testing a Logistic Regression document classifier for the classic [20 newsgroups corpus](https://github.com/apache/mahout/blob/master/examples/bin/classify-20newsgroups.sh) [4] is also available. +An example of training and testing a Logistic Regression document classifier for the classic [20 newsgroups corpus](https://github.com/apache/mahout/blob/master/examples/bin/classify-20newsgroups.sh) [4] is also available. ## Parallelization strategy The bad news is that SGD is an inherently sequential algorithm. The good news is that it is blazingly fast and thus it is not a problem for Mahout's -implementation to handle training sets of tens of millions of examples. +implementation to handle training sets of tens of millions of examples. With the down-sampling typical in many data-sets, this is equivalent to a dataset with billions of raw training examples. @@ -44,7 +44,7 @@ performance testing as your system runs. Often this means that you can stop training when a model reaches a target level of performance. The SGD framework includes classes to do on-line evaluation using cross validation (the CrossFoldLearner) and an evolutionary system to do learning -hyper-parameter optimization on the fly (the AdaptiveLogisticRegression). +hyper-parameter optimization on the fly (the AdaptiveLogisticRegression). The AdaptiveLogisticRegression system makes heavy use of threads to increase machine utilization. The way it works is that it runs 20 CrossFoldLearners in separate threads, each with slightly different @@ -127,4 +127,3 @@ Paul Komarek](http://repository.cmu.edu/cgi/viewcontent.cgi?article=1221&context [3] [SGD Bank Marketing Example](http://mahout.apache.org/users/classification/bankmarketing-example.html) [4] [SGD 20 newsgroups classification](https://github.com/apache/mahout/blob/master/examples/bin/classify-20newsgroups.sh) - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/mahout-collections.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/mahout-collections.md index 2ecce57d6b..a492a985ca 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/mahout-collections.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/mahout-collections.md @@ -13,7 +13,7 @@ redirect_from: The Mahout Collections library is a set of container classes that address some limitations of the standard collections in Java. [This presentation](http://domino.research.ibm.com/comm/research_people.nsf/pages/sevitsky.pubs.html/$FILE/oopsla08%20memory-efficient%20java%20slides.pdf) - describes a number of performance problems with the standard collections. + describes a number of performance problems with the standard collections. Mahout collections addresses two of the more glaring: the lack of support for primitive types and the lack of open hashing. @@ -53,9 +53,3 @@ have any additional storage usage for unused keys. The implementation of Mahout Collections is derived from [Cern Colt](http://acs.lbl.gov/~hoschek/colt/) . - - - - - - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/mlp.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/mlp.md index cb955fa661..545b37c624 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/mlp.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/mlp.md @@ -9,20 +9,20 @@ redirect_from: Multilayer Perceptron ===================== -A multilayer perceptron is a biologically inspired feed-forward network that can -be trained to represent a nonlinear mapping between input and output data. It +A multilayer perceptron is a biologically inspired feed-forward network that can +be trained to represent a nonlinear mapping between input and output data. It consists of multiple layers, each containing multiple artificial neuron units and -can be used for classification and regression tasks in a supervised learning approach. +can be used for classification and regression tasks in a supervised learning approach. Command line usage ------------------ The MLP implementation is currently located in the MapReduce-Legacy package. It -can be used with the following commands: +can be used with the following commands: # model training - $ bin/mahout org.apache.mahout.classifier.mlp.TrainMultilayerPerceptron + $ bin/mahout org.apache.mahout.classifier.mlp.TrainMultilayerPerceptron # model usage $ bin/mahout org.apache.mahout.classifier.mlp.RunMultilayerPerceptron @@ -67,14 +67,14 @@ To train our multilayer perceptron model from the command line, we call the foll The individual parameters are explained in the following. - `-i ./mrlegacy/src/test/resources/iris.csv` use the iris data set as input data -- `-sh` since the file `iris.csv` contains a header row, this row needs to be skipped +- `-sh` since the file `iris.csv` contains a header row, this row needs to be skipped - `-labels setosa versicolor virginica` we specify, which class labels should be learnt (which are the flower species in this case) - `-mo /tmp/model.model` specify where to store the model file - `-ls 4 8 3` we specify the structure and depth of our layers. The actual network structure can be seen in the figure below. - `-l 0.2` we set the learning rate to `0.2` - `-m 0.35` momemtum weight is set to `0.35` - `-r 0.0001` regularization weight is set to `0.0001` - + | | | |---|---| | The picture shows the architecture defined by the above command. The topolgy of the network is completely defined through the number of layers and units because in this implementation of the MLP every unit is fully connected to the units of the next and previous layer. Bias units are added automatically. | ![Multilayer perceptron network][mlp] | @@ -88,7 +88,7 @@ To test / run the multilayer perceptron classification on the trained model, we $ bin/mahout org.apache.mahout.classifier.mlp.RunMultilayerPerceptron \ -i ./mrlegacy/src/test/resources/iris.csv -sh -cr 0 3 \ -mo /tmp/model.model -o /tmp/labelResult.txt - + The individual parameters are explained in the following. @@ -98,31 +98,31 @@ The individual parameters are explained in the following. - `-mo /tmp/model.model` specify where the model file is stored - `-o /tmp/labelResult.txt` specify where the labeled output file will be stored -Implementation +Implementation -------------- The Multilayer Perceptron implementation is based on a more general Neural Network class. Command line support was added later on and provides a simple usage of the MLP as shown in the example. It is implemented to run on a single machine using stochastic gradient descent where the weights are updated using one datapoint at a time, resulting in a weight update of the form: $$ \vec{w}^{(t + 1)} = \vec{w}^{(t)} - n \Delta E_n(\vec{w}^{(t)}) $$ -where *a* is the activation of the unit. It is not yet possible to change the learning to more advanced methods using adaptive learning rates yet. +where *a* is the activation of the unit. It is not yet possible to change the learning to more advanced methods using adaptive learning rates yet. -The number of layers and units per layer can be specified manually and determines the whole topology with each unit being fully connected to the previous layer. A bias unit is automatically added to the input of every layer. +The number of layers and units per layer can be specified manually and determines the whole topology with each unit being fully connected to the previous layer. A bias unit is automatically added to the input of every layer. Currently, the logistic sigmoid is used as a squashing function in every hidden and output layer. It is of the form: $$ \frac{1}{1 + exp(-a)} $$ -The command line version **does not perform iterations** which leads to bad results on small datasets. Another restriction is, that the CLI version of the MLP only supports classification, since the labels have to be given explicitly when executing on the command line. +The command line version **does not perform iterations** which leads to bad results on small datasets. Another restriction is, that the CLI version of the MLP only supports classification, since the labels have to be given explicitly when executing on the command line. -A learned model can be stored and updated with new training instanced using the `--update` flag. Output of classification reults is saved as a .txt-file and only consists of the assigned labels. Apart from the command-line interface, it is possible to construct and compile more specialized neural networks using the API and interfaces in the mrlegacy package. +A learned model can be stored and updated with new training instanced using the `--update` flag. Output of classification reults is saved as a .txt-file and only consists of the assigned labels. Apart from the command-line interface, it is possible to construct and compile more specialized neural networks using the API and interfaces in the mrlegacy package. Theoretical Background ------------------------- -The *multilayer perceptron* was inspired by the biological structure of the brain where multiple neurons are connected and form columns and layers. Perceptual input enters this network through our sensory organs and is then further processed into higher levels. +The *multilayer perceptron* was inspired by the biological structure of the brain where multiple neurons are connected and form columns and layers. Perceptual input enters this network through our sensory organs and is then further processed into higher levels. The term multilayer perceptron is a little misleading since the *perceptron* is a special case of a single *artificial neuron* that can be used for simple computations [\[1\]][1]. The difference is that the perceptron uses a discontinous nonlinearity while for the MLP neurons that are implemented in mahout it is important to use continous nonlinearities. This is necessary for the implemented learning algorithm, where the error is propagated back from the output layer to the input layer and the weights of the connections are changed according to their contribution to the overall error. This algorithm is called backpropagation and uses gradient descent to update the weights. To compute the gradients we need continous nonlinearities. But let's start from the beginning! -The first layer of the MLP represents the input and has no other purpose than routing the input to every connected unit in a feed-forward fashion. Following layers are called hidden layers and the last layer serves the special purpose to determine the output. The activation of a unit *u* in a hidden layer is computed through a weighted sum of all inputs, resulting in +The first layer of the MLP represents the input and has no other purpose than routing the input to every connected unit in a feed-forward fashion. Following layers are called hidden layers and the last layer serves the special purpose to determine the output. The activation of a unit *u* in a hidden layer is computed through a weighted sum of all inputs, resulting in $$ a_j = \sum_{i=1}^{D} w_{ji}^{(l)} x_i + w_{j0}^{(l)} $$ This computes the activation *a* for neuron *j* where *w* is the weight from neuron *i* to neuron *j* in layer *l*. The last part, where *i = 0* is called the bias and can be used as an offset, independent from the input. @@ -131,15 +131,15 @@ The activation is then transformed by the aforementioned differentiable, nonline For different units or layers, different activation functions can be used to obtain different behaviors. Especially in the output layer, the activation function can be chosen to obtain the output value *y*, depending on the learning problem: $$ y_k = \sigma (a_k) $$ -If the learning problem is a linear regression task, sigma can be chosen to be the identity function. In case of classification problems, the choice of the squashing functions depends on the exact task at hand and often softmax activation functions are used. +If the learning problem is a linear regression task, sigma can be chosen to be the identity function. In case of classification problems, the choice of the squashing functions depends on the exact task at hand and often softmax activation functions are used. The equation for a MLP with three layers (one input, one hidden and one output) is then given by -$$ y_k(\vec{x}, \vec{w}) = h \left( \sum_{j=1}^{M} w_{kj}^{(2)} h \left( \sum_{i=1}^{D} w_{ji}^{(1)} x_i + w_{j0}^{(1)} \right) + w_{k0}^{(2)} \right) $$ +$$ y_k(\vec{x}, \vec{w}) = h \left( \sum_{j=1}^{M} w_{kj}^{(2)} h \left( \sum_{i=1}^{D} w_{ji}^{(1)} x_i + w_{j0}^{(1)} \right) + w_{k0}^{(2)} \right) $$ where *h* indicates the respective squashing function that is used in the units of a layer. *M* and *D* specify the number of incoming connections to a unit and we can see that the input to the first layer (hidden layer) is just the original input *x* whereas the input into the second layer (output layer) is the transformed output of layer one. The output *y* of unit *k* is therefore given by the above equation and depends on the input *x* and the weight vector *w*. This shows us, that the parameter that we can optimize during learning is *w* since we can not do anything about the input *x*. To facilitate the following steps, we can include the bias-terms into the weight vector and correct for the indices by adding another dimension with the value 1 to the input vector. The bias is a constant factor that is added to the weighted sum and that serves as a scaling factor of the nonlinear transformation. Including it into the weight vector leads to: -$$ y_k(\vec{x}, \vec{w}) = h \left( \sum_{j=0}^{M} w_{kj}^{(2)} h \left( \sum_{i=0}^{D} w_{ji}^{(1)} x_i \right) \right) $$ +$$ y_k(\vec{x}, \vec{w}) = h \left( \sum_{j=0}^{M} w_{kj}^{(2)} h \left( \sum_{i=0}^{D} w_{ji}^{(1)} x_i \right) \right) $$ The previous paragraphs described how the MLP transforms a given input into some output using a combination of different nonlinear functions. Of course what we really want is to learn the structure of our data so that we can feed data with unknown labels into the network and get the estimated target labels *t*. To achieve this, we have to train our network. In this context, training means optimizing some function such that the error between the real labels *y* and the network-output *t* becomes smallest. We have seen in the previous pragraph, that our only knob to change is the weight vector *w*, making the function to be optimized a function of *w*. For simplicitly and because it is widely used, we choose the so called *sum-of-squares* error function as an example that is given by @@ -147,7 +147,7 @@ $$ E(\vec{w}) = \frac{1}{2} \sum_{n=1}^N \left( y(\vec{x}_n, \vec{w}) - t_n \rig The goal is to minimize this function and thereby increase the performance of our model. A common method to achieve this is to use gradient descent and the so called technique of *backpropagation* where the goal is to compute the contribution of every unit to the overall error and changing the weight according to this contribution and into the direction of the gradient of the error function at this particular unit. In the following we try to give a short overview of the model training with gradient descent and backpropagation. A more detailed example can be found in [\[3\]][3] where much of this information is taken from. -The problem with minimizing the error function is that the error can only be computed at the output layers where we get *t*, but we want to update all the weights of all the units. Therefore we use the technique of backpropagation to propagate the error, that we first compute at the output layer, back to the units of the previous layers. For this approach we also need to compute the gradients of the activation function. +The problem with minimizing the error function is that the error can only be computed at the output layers where we get *t*, but we want to update all the weights of all the units. Therefore we use the technique of backpropagation to propagate the error, that we first compute at the output layer, back to the units of the previous layers. For this approach we also need to compute the gradients of the activation function. Weights are then updated with a small step in the direction of the negative gradient, regulated by the learning rate *n* such that we arrive at the formula for weight update: @@ -170,4 +170,3 @@ References \[2\] http://en.wikipedia.org/wiki/Sigmoid_function \[3\] [Christopher M. Bishop: Pattern Recognition and Machine Learning, Springer 2009](http://research.microsoft.com/en-us/um/people/cmbishop/prml/) - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/partial-implementation.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/partial-implementation.md index 6683d5f316..b1326b1d7f 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/partial-implementation.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/partial-implementation.md @@ -44,7 +44,7 @@ $HADOOP_HOME/bin/hadoop fs -put testdata{code} * In $MAHOUT_HOME/ run: {code}mvn clean install -DskipTests{code} -## Generate a file descriptor for the dataset: +## Generate a file descriptor for the dataset: run the following command: $HADOOP_HOME/bin/hadoop jar @@ -118,7 +118,7 @@ matrix Correctly Classified Instances : 17657 78.3224% Incorrectly Classified Instances : 4887 21.6776% Total Classified Instances : 22544 - + ======================================================= Confusion Matrix ------------------------------------------------------- @@ -139,7 +139,7 @@ the output will be a directory 'predictions' containing two files The "Decision Forest" code is still "a work in progress", many features are still missing. Here is a list of some known issues: * For now, the training does not support multiple input files. The input -dataset must be one single file (this support will be available with the upcoming release). +dataset must be one single file (this support will be available with the upcoming release). Classifying new data does support multiple input files. * The tree building is done when each mapper.close() method is called. diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/restricted-boltzmann-machines.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/restricted-boltzmann-machines.md index e3de6cd644..b2666a075d 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/restricted-boltzmann-machines.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/classification/restricted-boltzmann-machines.md @@ -7,17 +7,17 @@ redirect_from: --- NOTE: This implementation is a Work-In-Progress, at least till September, -2010. +2010. The JIRA issue is [here](https://issues.apache.org/jira/browse/MAHOUT-375) -. +. ### Boltzmann Machines Boltzmann Machines are a type of stochastic neural networks that closely resemble physical processes. They define a network of units with an overall energy that is evolved over a period of time, until it reaches thermal -equilibrium. +equilibrium. However, the convergence speed of Boltzmann machines that have unconstrained connectivity is low. @@ -29,7 +29,7 @@ sense that connections between hidden units of a single layer are _not_ allowed. In addition, stacking multiple RBM's is also feasible, with the activities of the hidden units forming the base for a higher-level RBM. The combination of these two features renders RBM's highly usable for -parallelization. +parallelization. In the Netflix Prize, RBM's offered distinctly orthogonal predictions to SVD and k-NN approaches, and contributed immensely to the final solution. diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/canopy-clustering.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/canopy-clustering.md index bfc65f209f..0f8e37ca2f 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/canopy-clustering.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/canopy-clustering.md @@ -33,7 +33,7 @@ distance measurements can be significantly reduced by ignoring points outside of the initial canopies. **WARNING**: Canopy is deprecated in the latest release and will be removed once streaming k-means becomes stable enough. - + ## Strategy for parallelization @@ -72,7 +72,7 @@ to produce canopy centroids which are output, using a constant key centroids and again applies the canopy measure and thresholds to produce a final set of canopy centroids which is output (i.e. clustering the cluster centroids). The reducer output format is: SequenceFile(Text, Canopy) with -the _key_ encoding the canopy identifier. +the _key_ encoding the canopy identifier. ### Clustering phase @@ -171,7 +171,7 @@ The points are generated as follows: sd=0.1 In the first image, the points are plotted and the 3-sigma boundaries of -their generator are superimposed. +their generator are superimposed. ![sample data](../../images/SampleData.png) @@ -189,4 +189,3 @@ enough to iterate runs using different T1, T2 parameters and display thresholds. ![canopy](../../images/Canopy10.png) - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/cluster-dumper.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/cluster-dumper.md index 920b003fd1..d711f1e932 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/cluster-dumper.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/cluster-dumper.md @@ -22,7 +22,7 @@ you can run clusterdumper in 2 modes: 1. Hadoop Environment -1. Standalone Java Program +1. Standalone Java Program @@ -37,38 +37,38 @@ executed the [synthetic control example ](clustering-of-synthetic-control-data.h and want to analyze the output, you can execute the `mahout clusterdumper` utility from the command line. #### CLI options: - --help Print out help + --help Print out help --input (-i) input The directory containing Sequence - Files for the Clusters + Files for the Clusters --output (-o) output The output file. If not specified, dumps to the console. --outputFormat (-of) outputFormat The optional output format to write - the results as. Options: TEXT, CSV, or GRAPH_ML - --substring (-b) substring The number of chars of the - asFormatString() to print - --pointsDir (-p) pointsDir The directory containing points + the results as. Options: TEXT, CSV, or GRAPH_ML + --substring (-b) substring The number of chars of the + asFormatString() to print + --pointsDir (-p) pointsDir The directory containing points sequence files mapping input vectors - to their cluster. If specified, - then the program will output the - points associated with a cluster + to their cluster. If specified, + then the program will output the + points associated with a cluster --dictionary (-d) dictionary The dictionary file. - --dictionaryType (-dt) dictionaryType The dictionary file type + --dictionaryType (-dt) dictionaryType The dictionary file type (text|sequencefile) --distanceMeasure (-dm) distanceMeasure The classname of the DistanceMeasure. Default is SquaredEuclidean. - --numWords (-n) numWords The number of top terms to print + --numWords (-n) numWords The number of top terms to print --tempDir tempDir Intermediate output directory --startPhase startPhase First phase to run --endPhase endPhase Last phase to run --evaluate (-e) Run ClusterEvaluator and CDbwEvaluator over the input. The output will be appended to the rest of - the output at the end. + the output at the end. -### Standalone Java Program +### Standalone Java Program Run the clusterdump utility as follows as a standalone Java Program through Eclipse. To execute ClusterDumper.java, - + * Under mahout-utils, Right-Click on ClusterDumper.java * Choose Run-As, Run Configurations * On the left menu, click on Java Application @@ -80,15 +80,15 @@ Run the clusterdump utility as follows as a standalone Java Program through Ecli In the arguments tab, specify the below arguments - --seqFileDir /examples/output/clusters-10 - --pointsDir /examples/output/clusteredPoints + --seqFileDir /examples/output/clusters-10 + --pointsDir /examples/output/clusteredPoints --output /examples/output/clusteranalyze.txt replace with the actual path of your $MAHOUT_HOME * Hit run to execute the ClusterDumper using Eclipse. Setting breakpoints etc should just work fine. - + Reading the output file - + This will output the clusters into a file called clusteranalyze.txt inside $MAHOUT_HOME/examples/output Sample data will look like diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/expectation-maximization.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/expectation-maximization.md index f31f6624e1..80ecab790a 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/expectation-maximization.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/expectation-maximization.md @@ -60,4 +60,3 @@ exponentially small influence. ## Map/Reduce Implementation - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/fuzzy-k-means.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/fuzzy-k-means.md index 97efe64a2e..c429f0e669 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/fuzzy-k-means.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/fuzzy-k-means.md @@ -36,8 +36,8 @@ Similar to K-Means, the program doesn't modify the input directories. And for every iteration, the cluster output is stored in a directory cluster-N. The code has set number of reduce tasks equal to number of map tasks. So, those many part-0 - - + + Files are created in clusterN directory. The code uses driver/mapper/combiner/reducer as follows: @@ -75,7 +75,7 @@ converged clusters with 'Vn' clusterId. The Fuzzy k-Means clustering algorithm may be run using a command-line invocation on FuzzyKMeansDriver.main or by making a Java call to -FuzzyKMeansDriver.run(). +FuzzyKMeansDriver.run(). Invocation using the command line takes the form: @@ -142,7 +142,7 @@ SequenceFile(IntWritable, WeightedVectorWritable). The IntWritable _key_ is the clusterId. The WeightedVectorWritable _value_ is a bean containing a double _weight_ and a VectorWritable _vector_ where the weights are computed as 1/(1+distance) where the distance is between the cluster center -and the vector using the chosen DistanceMeasure. +and the vector using the chosen DistanceMeasure. # Examples @@ -163,7 +163,7 @@ The points are generated as follows: sd=0.1 In the first image, the points are plotted and the 3-sigma boundaries of -their generator are superimposed. +their generator are superimposed. ![fuzzy]({{ BASE_PATH }}/assets/img/SampleData.png) diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/hierarchical-clustering.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/hierarchical-clustering.md index b97a6a91b3..e160f37040 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/hierarchical-clustering.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/hierarchical-clustering.md @@ -9,8 +9,7 @@ Hierarchical clustering is the process or finding bigger clusters, and also the smaller clusters inside the bigger clusters. In Apache Mahout, separate algorithms can be used for finding clusters at -different levels. +different levels. See [Top Down Clustering](https://cwiki.apache.org/confluence/display/MAHOUT/Top+Down+Clustering) . - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/latent-dirichlet-allocation.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/latent-dirichlet-allocation.md index 144143e229..b8f6676aae 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/latent-dirichlet-allocation.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/latent-dirichlet-allocation.md @@ -13,7 +13,7 @@ Latent Dirichlet Allocation (Blei et al, 2003) is a powerful learning algorithm for automatically and jointly clustering words into "topics" and documents into mixtures of topics. It has been successfully applied to model change in scientific fields over time (Griffiths and Steyvers, 2004; -Hall, et al. 2008). +Hall, et al. 2008). A topic model is, roughly, a hierarchical Bayesian model that associates with each document a probability distribution over "topics", which are in @@ -98,7 +98,7 @@ test_set_percentage>0> \ Topic smoothing should generally be about 50/K, where K is the number of topics. The number of words in the vocabulary can be an upper bound, though -it shouldn't be too high (for memory concerns). +it shouldn't be too high (for memory concerns). Choosing the number of topics is more art than science, and it's recommended that you try several values. @@ -124,7 +124,7 @@ An example is located in mahout/examples/bin/build-reuters.sh. The script automatically downloads the Reuters-21578 corpus, builds a Lucene index and converts the Lucene index to vectors. By uncommenting the last two lines you can then cause it to run LDA on the vectors and finally print the -resultant topics to the console. +resultant topics to the console. To adapt the example yourself, you should note that Lucene has specialized support for Reuters, and that building your own index will require some @@ -140,7 +140,7 @@ document, inferring the posterior probability of each topic for each word in each document. We then take the sufficient statistics and emit them in the form of (log) pseudo-counts for each word in each topic. The M-Step is simply to sum these together and (log) normalize them so that we have a -distribution over the entire vocabulary of the corpus for each topic. +distribution over the entire vocabulary of the corpus for each topic. In implementation, the E-Step is implemented in the Map, and the M-Step is executed in the reduce step, with the final normalization happening as a diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/llr---log-likelihood-ratio.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/llr---log-likelihood-ratio.md index 06ff90ef3d..c1a76eedcd 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/llr---log-likelihood-ratio.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/llr---log-likelihood-ratio.md @@ -26,11 +26,11 @@ positive and false negative rates. The LLR is typically far more suitable in the case of small than many other measures such as Pearson's correlation, Pearson's chi squared statistic or z statistics. The LLR as stated does not, however, make any use of rating data which can limit its -applicability in problems such as the Netflix competition. +applicability in problems such as the Netflix competition. The actual value of the LLR is not usually very helpful other than as a way of ordering pairs of items. As such, it is often used to determine a -sparse set of coefficients to be estimated by other means such as TF-IDF. +sparse set of coefficients to be estimated by other means such as TF-IDF. Since the actual estimation of these coefficients can be done in a way that is independent of the training data such as by general corpus statistics, and since the ordering imposed by the LLR is relatively robust to counting @@ -38,9 +38,8 @@ fluctuation, this technique can provide very strong results in very sparse problems where the potential number of features vastly out-numbers the number of training examples and where features are highly interdependent. - See Also: + See Also: * [Blog post "surprise and coincidence"](http://tdunning.blogspot.com/2008/03/surprise-and-coincidence.html) * [G-Test](http://en.wikipedia.org/wiki/G-test) * [Likelihood Ratio Test](http://en.wikipedia.org/wiki/Likelihood-ratio_test) - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/streaming-k-means.md b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/streaming-k-means.md index 7bd9f62548..774d040db1 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/streaming-k-means.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/map-reduce/clustering/streaming-k-means.md @@ -6,18 +6,18 @@ redirect_from: - /docs/latest/algorithms/map-reduce/clustering/streaming-k-means.html --- -# *StreamingKMeans* algorithm +# *StreamingKMeans* algorithm The *StreamingKMeans* algorithm is a variant of Algorithm 1 from [Shindler et al][1] and consists of two steps: - 1. Streaming step - 2. BallKMeans step. + 1. Streaming step + 2. BallKMeans step. -The streaming step is a randomized algorithm that makes one pass through the data and -produces as many centroids as it determines is optimal. This step can be viewed as -a preparatory dimensionality reduction. If the size of the data stream is *n* and the -expected number of clusters is *k*, the streaming step will produce roughly *k\*log(n)* -clusters that will be passed on to the BallKMeans step which will further reduce the +The streaming step is a randomized algorithm that makes one pass through the data and +produces as many centroids as it determines is optimal. This step can be viewed as +a preparatory dimensionality reduction. If the size of the data stream is *n* and the +expected number of clusters is *k*, the streaming step will produce roughly *k\*log(n)* +clusters that will be passed on to the BallKMeans step which will further reduce the number of clusters down to *k*. BallKMeans is a randomized Lloyd-type algorithm that has been studied in detail, see [Ostrovsky et al][2]. @@ -27,35 +27,35 @@ has been studied in detail, see [Ostrovsky et al][2]. ### Overview -The streaming step is a derivative of the streaming -portion of Algorithm 1 in [Shindler et al][1]. The main difference between the two is that -Algorithm 1 of [Shindler et al][1] assumes -the knowledge of the size of the data stream and uses it to set a key parameter -for the algorithm. More precisely, the initial *distanceCutoff* (defined below), which is -denoted by *f* in [Shindler et al][1], is set to *1/(k(1+log(n))*. The *distanceCutoff* influences the number of clusters that the algorithm -will produce. -In contrast, Mahout implementation does not require the knowledge of the size of the -data stream. Instead, it dynamically re-evaluates the parameters that depend on the size -of the data stream at runtime as more and more data is processed. In particular, -the parameter *numClusters* (defined below) changes its value as the data is processed. +The streaming step is a derivative of the streaming +portion of Algorithm 1 in [Shindler et al][1]. The main difference between the two is that +Algorithm 1 of [Shindler et al][1] assumes +the knowledge of the size of the data stream and uses it to set a key parameter +for the algorithm. More precisely, the initial *distanceCutoff* (defined below), which is +denoted by *f* in [Shindler et al][1], is set to *1/(k(1+log(n))*. The *distanceCutoff* influences the number of clusters that the algorithm +will produce. +In contrast, Mahout implementation does not require the knowledge of the size of the +data stream. Instead, it dynamically re-evaluates the parameters that depend on the size +of the data stream at runtime as more and more data is processed. In particular, +the parameter *numClusters* (defined below) changes its value as the data is processed. ###Parameters - - **numClusters** (int): Conceptually, *numClusters* represents the algorithm's guess at the optimal -number of clusters it is shooting for. In particular, *numClusters* will increase at run -time as more and more data is processed. Note that •numClusters• is not the number of clusters that the algorithm will produce. Also, *numClusters* should not be set to the final number of clusters that we expect to receive as the output of *StreamingKMeans*. - - **distanceCutoff** (double): a parameter representing the value of the distance between a point and + - **numClusters** (int): Conceptually, *numClusters* represents the algorithm's guess at the optimal +number of clusters it is shooting for. In particular, *numClusters* will increase at run +time as more and more data is processed. Note that •numClusters• is not the number of clusters that the algorithm will produce. Also, *numClusters* should not be set to the final number of clusters that we expect to receive as the output of *StreamingKMeans*. + - **distanceCutoff** (double): a parameter representing the value of the distance between a point and its closest centroid after which -the new point will definitely be assigned to a new cluster. *distanceCutoff* can be thought -of as an estimate of the variable *f* from Shindler et al. The default initial value for -*distanceCutoff* is *1.0/numClusters* and *distanceCutoff* grows as a geometric progression with -common ratio *beta* (see below). - - **beta** (double): a constant parameter that controls the growth of *distanceCutoff*. If the initial setting of *distanceCutoff* is *d0*, *distanceCutoff* will grow as the geometric progression with initial term *d0* and common ratio *beta*. The default value for *beta* is 1.3. - - **clusterLogFactor** (double): a constant parameter such that *clusterLogFactor* *log(numProcessedPoints)* is the runtime estimate of the number of clusters to be produced by the streaming step. If the final number of clusters (that we expect *StreamingKMeans* to output) is *k*, *clusterLogFactor* can be set to *k*. - - **clusterOvershoot** (double): a constant multiplicative slack factor that slows down the collapsing of clusters. The default value is 2. +the new point will definitely be assigned to a new cluster. *distanceCutoff* can be thought +of as an estimate of the variable *f* from Shindler et al. The default initial value for +*distanceCutoff* is *1.0/numClusters* and *distanceCutoff* grows as a geometric progression with +common ratio *beta* (see below). + - **beta** (double): a constant parameter that controls the growth of *distanceCutoff*. If the initial setting of *distanceCutoff* is *d0*, *distanceCutoff* will grow as the geometric progression with initial term *d0* and common ratio *beta*. The default value for *beta* is 1.3. + - **clusterLogFactor** (double): a constant parameter such that *clusterLogFactor* *log(numProcessedPoints)* is the runtime estimate of the number of clusters to be produced by the streaming step. If the final number of clusters (that we expect *StreamingKMeans* to output) is *k*, *clusterLogFactor* can be set to *k*. + - **clusterOvershoot** (double): a constant multiplicative slack factor that slows down the collapsing of clusters. The default value is 2. -###Algorithm +###Algorithm The algorithm processes the data one-by-one and makes only one pass through the data. The first point from the data stream will form the centroid of the first cluster (this designation may change as more points are processed). Suppose there are *r* clusters at one point and a new point *p* is being processed. The new point can either be added to one of the existing *r* clusters or become a new cluster. To decide: @@ -63,7 +63,7 @@ The first point from the data stream will form the centroid of the first cluster - let *c* be the closest cluster to point *p* - let *d* be the distance between *c* and *p* - if *d > distanceCutoff*, create a new cluster from *p* (*p* is too far away from the clusters to be part of any one of them) - - else (*d <= distanceCutoff*), create a new cluster with probability *d / distanceCutoff* (the probability of creating a new cluster increases as *d* increases). + - else (*d <= distanceCutoff*), create a new cluster with probability *d / distanceCutoff* (the probability of creating a new cluster increases as *d* increases). There will be either *r* or *r+1* clusters after processing a new point. @@ -73,12 +73,12 @@ are treated as data points and are re-clustered (collapsed). This tends to make ## BallKMeans step --- ### Overview -The algorithm is a Lloyd-type algorithm that takes a set of weighted vectors and returns k centroids, see [Ostrovsky et al][2] for details. The algorithm has two stages: - - 1. Seeding - 2. Ball k-means +The algorithm is a Lloyd-type algorithm that takes a set of weighted vectors and returns k centroids, see [Ostrovsky et al][2] for details. The algorithm has two stages: + + 1. Seeding + 2. Ball k-means -The seeding stage is an initial guess of where the centroids should be. The initial guess is improved using the ball k-means stage. +The seeding stage is an initial guess of where the centroids should be. The initial guess is improved using the ball k-means stage. ### Parameters @@ -97,71 +97,71 @@ The seeding stage is an initial guess of where the centroids should be. The init * **numRuns** (int): This is the number of runs to perform. The solution of lowest cost is returned. The default is 1 run. ###Algorithm -The algorithm can be instructed to take multiple independent runs (using the *numRuns* parameter) and the algorithm will select the best solution (i.e., the one with the lowest cost). In practice, one run is sufficient to find a good solution. +The algorithm can be instructed to take multiple independent runs (using the *numRuns* parameter) and the algorithm will select the best solution (i.e., the one with the lowest cost). In practice, one run is sufficient to find a good solution. Each run operates as follows: a seeding procedure is used to select k centroids, and then ball k-means is run iteratively to refine the solution. -The seeding procedure can be set to either 'uniformly at random' or 'k-means++' using *kMeansPlusPlusInit* boolean variable. Seeding with k-means++ involves more computation but offers better results in practice. - +The seeding procedure can be set to either 'uniformly at random' or 'k-means++' using *kMeansPlusPlusInit* boolean variable. Seeding with k-means++ involves more computation but offers better results in practice. + Each iteration of ball k-means runs as follows: 1. Clusters are formed by assigning each datapoint to the nearest centroid -2. The centers of mass of the trimmed clusters (see *trimFraction* parameter above) become the new centroids +2. The centers of mass of the trimmed clusters (see *trimFraction* parameter above) become the new centroids The data may be partitioned into a test set and a training set (see *testProbability*). The seeding procedure and ball k-means run on the training set. The cost is computed on the test set. -##Usage of *StreamingKMeans* - - bin/mahout streamingkmeans - -i - -o - -ow - -k - -km - -e - -mi - -tf - -ri - -iw - -testp - -nbkm - -dm - -sc - -np - -s - -rskm - -xm - -h - --tempDir - --startPhase - --endPhase +##Usage of *StreamingKMeans* + + bin/mahout streamingkmeans + -i + -o + -ow + -k + -km + -e + -mi + -tf + -ri + -iw + -testp + -nbkm + -dm + -sc + -np + -s + -rskm + -xm + -h + --tempDir + --startPhase + --endPhase ###Details on Job-Specific Options: - - * `--input (-i) `: Path to job input directory. - * `--output (-o) `: The directory pathname for output. + + * `--input (-i) `: Path to job input directory. + * `--output (-o) `: The directory pathname for output. * `--overwrite (-ow)`: If present, overwrite the output directory before running job. - * `--numClusters (-k) `: The k in k-Means. Approximately this many clusters will be generated. - * `--estimatedNumMapClusters (-km) `: The estimated number of clusters to use for the Map phase of the job when running StreamingKMeans. This should be around k \* log(n), where k is the final number of clusters and n is the total number of data points to cluster. - * `--estimatedDistanceCutoff (-e) `: The initial estimated distance cutoff between two points for forming new clusters. If no value is given, it's estimated from the data set - * `--maxNumIterations (-mi) `: The maximum number of iterations to run for the BallKMeans algorithm used by the reducer. If no value is given, defaults to 10. - * `--trimFraction (-tf) `: The 'ball' aspect of ball k-means means that only the closest points to the centroid will actually be used for updating. The fraction of the points to be used is those points whose distance to the center is within trimFraction \* distance to the closest other center. If no value is given, defaults to 0.9. + * `--numClusters (-k) `: The k in k-Means. Approximately this many clusters will be generated. + * `--estimatedNumMapClusters (-km) `: The estimated number of clusters to use for the Map phase of the job when running StreamingKMeans. This should be around k \* log(n), where k is the final number of clusters and n is the total number of data points to cluster. + * `--estimatedDistanceCutoff (-e) `: The initial estimated distance cutoff between two points for forming new clusters. If no value is given, it's estimated from the data set + * `--maxNumIterations (-mi) `: The maximum number of iterations to run for the BallKMeans algorithm used by the reducer. If no value is given, defaults to 10. + * `--trimFraction (-tf) `: The 'ball' aspect of ball k-means means that only the closest points to the centroid will actually be used for updating. The fraction of the points to be used is those points whose distance to the center is within trimFraction \* distance to the closest other center. If no value is given, defaults to 0.9. * `--randomInit` (`-ri`) Whether to use k-means++ initialization or random initialization of the seed centroids. Essentially, k-means++ provides better clusters, but takes longer, whereas random initialization takes less time, but produces worse clusters, and tends to fail more often and needs multiple runs to compare to k-means++. If set, uses the random initialization. - * `--ignoreWeights (-iw)`: Whether to correct the weights of the centroids after the clustering is done. The weights end up being wrong because of the trimFraction and possible train/test splits. In some cases, especially in a pipeline, having an accurate count of the weights is useful. If set, ignores the final weights. - * `--testProbability (-testp) `: A double value between 0 and 1 that represents the percentage of points to be used for 'testing' different clustering runs in the final BallKMeans step. If no value is given, defaults to 0.1 - * `--numBallKMeansRuns (-nbkm) `: Number of BallKMeans runs to use at the end to try to cluster the points. If no value is given, defaults to 4 - * `--distanceMeasure (-dm) `: The classname of the DistanceMeasure. Default is SquaredEuclidean. - * `--searcherClass (-sc) `: The type of searcher to be used when performing nearest neighbor searches. Defaults to ProjectionSearch. - * `--numProjections (-np) `: The number of projections considered in estimating the distances between vectors. Only used when the distance measure requested is either ProjectionSearch or FastProjectionSearch. If no value is given, defaults to 3. - * `--searchSize (-s) `: In more efficient searches (non BruteSearch), not all distances are calculated for determining the nearest neighbors. The number of elements whose distances from the query vector is actually computer is proportional to searchSize. If no value is given, defaults to 1. - * `--reduceStreamingKMeans (-rskm)`: There might be too many intermediate clusters from the mapper to fit into memory, so the reducer can run another pass of StreamingKMeans to collapse them down to a fewer clusters. - * `--method (-xm)` method The execution method to use: sequential or mapreduce. Default is mapreduce. - * `-- help (-h)`: Print out help + * `--ignoreWeights (-iw)`: Whether to correct the weights of the centroids after the clustering is done. The weights end up being wrong because of the trimFraction and possible train/test splits. In some cases, especially in a pipeline, having an accurate count of the weights is useful. If set, ignores the final weights. + * `--testProbability (-testp) `: A double value between 0 and 1 that represents the percentage of points to be used for 'testing' different clustering runs in the final BallKMeans step. If no value is given, defaults to 0.1 + * `--numBallKMeansRuns (-nbkm) `: Number of BallKMeans runs to use at the end to try to cluster the points. If no value is given, defaults to 4 + * `--distanceMeasure (-dm) `: The classname of the DistanceMeasure. Default is SquaredEuclidean. + * `--searcherClass (-sc) `: The type of searcher to be used when performing nearest neighbor searches. Defaults to ProjectionSearch. + * `--numProjections (-np) `: The number of projections considered in estimating the distances between vectors. Only used when the distance measure requested is either ProjectionSearch or FastProjectionSearch. If no value is given, defaults to 3. + * `--searchSize (-s) `: In more efficient searches (non BruteSearch), not all distances are calculated for determining the nearest neighbors. The number of elements whose distances from the query vector is actually computer is proportional to searchSize. If no value is given, defaults to 1. + * `--reduceStreamingKMeans (-rskm)`: There might be too many intermediate clusters from the mapper to fit into memory, so the reducer can run another pass of StreamingKMeans to collapse them down to a fewer clusters. + * `--method (-xm)` method The execution method to use: sequential or mapreduce. Default is mapreduce. + * `-- help (-h)`: Print out help * `--tempDir `: Intermediate output directory. - * `--startPhase ` First phase to run. - * `--endPhase ` Last phase to run. + * `--startPhase ` First phase to run. + * `--endPhase ` Last phase to run. ##References diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/preprocessors/AsFactor.md b/website/Map_Reduce_Folder/docs/latest/algorithms/preprocessors/AsFactor.md index 2daa683062..e4ebf494b0 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/preprocessors/AsFactor.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/preprocessors/AsFactor.md @@ -11,13 +11,13 @@ redirect_from: The `AsFactor` preprocessor is used to turn the integer values of the columns into sparse vectors where the value is 1 at the index that corresponds to the 'category' of that column. This is also known as "One Hot Encoding" in many other - packages. - + packages. + ### Parameters `AsFactor` takes no parameters. - + ### Example ```scala @@ -33,4 +33,3 @@ val factorizer: AsFactorModel = new AsFactor().fit(A) val factoredA = factorizer.transform(A) ``` - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/preprocessors/MeanCenter.md b/website/Map_Reduce_Folder/docs/latest/algorithms/preprocessors/MeanCenter.md index f5a8fc3656..05811b8457 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/preprocessors/MeanCenter.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/preprocessors/MeanCenter.md @@ -8,7 +8,7 @@ redirect_from: ### About -`MeanCenter` centers values about the column mean. +`MeanCenter` centers values about the column mean. ### Parameters @@ -26,6 +26,3 @@ val scaler: MeanCenterModel = new MeanCenter().fit(A) val centeredA = scaler.transform(A) ``` - - - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/preprocessors/StandardScaler.md b/website/Map_Reduce_Folder/docs/latest/algorithms/preprocessors/StandardScaler.md index a4dbedb37d..f1fc060ec1 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/preprocessors/StandardScaler.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/preprocessors/StandardScaler.md @@ -8,13 +8,13 @@ redirect_from: ### About -`StandardScaler` centers the values of each column to their mean, and scales them to unit variance. +`StandardScaler` centers the values of each column to their mean, and scales them to unit variance. #### Relation to the `scale` function in R-base The `StandardScaler` is the equivelent of the R-base function [`scale`](https://stat.ethz.ch/R-manual/R-devel/library/base/html/scale.html) with -one noteable tweek. R's `scale` function (indeed all of R) calculates standard deviation with 1 degree of freedom, Mahout +one noteable tweek. R's `scale` function (indeed all of R) calculates standard deviation with 1 degree of freedom, Mahout (like many other statistical packages aimed at larger data sets) does not make this adjustment. In larger datasets the difference -is trivial, however when testing the function on smaller datasets the practicioner may be confused by the discrepency. +is trivial, however when testing the function on smaller datasets the practicioner may be confused by the discrepency. To verify this function against R on an arbitrary matrix, use the following form in R to "undo" the degrees of freedom correction. ```R @@ -41,5 +41,3 @@ val scaler: StandardScalerModel = new StandardScaler().fit(A) val scaledA = scaler.transform(A) ``` - - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/preprocessors/index.md b/website/Map_Reduce_Folder/docs/latest/algorithms/preprocessors/index.md index b915b4912b..4e1aeaa11a 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/preprocessors/index.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/preprocessors/index.md @@ -11,4 +11,3 @@ redirect_from: [Mean Center](MeanCenter.html) - For mean centering [StandardScaler](StandardScaler.html) - For mean centering and unit variance - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/recommenders/cco.md b/website/Map_Reduce_Folder/docs/latest/algorithms/recommenders/cco.md index a136782b0e..88005fead5 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/recommenders/cco.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/recommenders/cco.md @@ -8,7 +8,7 @@ redirect_from: # Building a Correlated Cross-Occurrence (CCO) Recommenders with the Mahout CLI -Mahout's CCO algorithm is one of a new breed of "Multimodal" recommenders that can use input of many types in very flexible ways. +Mahout's CCO algorithm is one of a new breed of "Multimodal" recommenders that can use input of many types in very flexible ways. Mahout provides several important building blocks for creating recommendations using Spark. *spark-itemsimilarity* can be used to create "other people also liked these things" type recommendations and paired with a search engine can personalize recommendations for individual users. *spark-rowsimilarity* can provide non-personalized content based recommendations and when paired with a search engine can be used to personalize content based recommendations. @@ -16,7 +16,7 @@ Mahout provides several important building blocks for creating recommendations u This is a simplified Lambda architecture with Mahout's *spark-itemsimilarity* playing the batch model building role and a search engine playing the realtime serving role. -You will create two collections, one for user history and one for item "indicators". Indicators are user interactions that lead to the wished for interaction. So for example if you wish a user to purchase something and you collect all users purchase interactions *spark-itemsimilarity* will create a purchase indicator from them. But you can also use other user interactions in a cross-cooccurrence calculation, to create purchase indicators. +You will create two collections, one for user history and one for item "indicators". Indicators are user interactions that lead to the wished for interaction. So for example if you wish a user to purchase something and you collect all users purchase interactions *spark-itemsimilarity* will create a purchase indicator from them. But you can also use other user interactions in a cross-cooccurrence calculation, to create purchase indicators. User history is used as a query on the item collection with its cooccurrence and cross-cooccurrence indicators (there may be several indicators). The primary interaction or indicator is picked to be the thing you want to recommend, other action / indicators are believed to be correlated but may not indicate exactly the same user intent. For instance in an ecom recommender a purchase is a very good primary action / indicator, but you may also know product detail-views, or additions-to-wishlists. These can be considered secondary actions / indicators which may all be used to calculate cross-cooccurrence indicators. The user history that forms the recommendations query will contain recorded primary and secondary indicators all targeted towards the correct indicator fields. @@ -33,20 +33,20 @@ Below are the command line jobs but the drivers and associated code can also be ## 1. spark-itemsimilarity *spark-itemsimilarity* is the Spark counterpart of the of the Mahout mapreduce job called *itemsimilarity*. It takes in elements of interactions, which have userID, itemID, and optionally a value. It will produce one of more indicator matrices created by comparing every user's interactions with every other user. The indicator matrix is an item x item matrix where the values are log-likelihood ratio strengths. For the legacy mapreduce version, there were several possible similarity measures but these are being deprecated in favor of LLR because in practice it performs the best. -Mahout's mapreduce version of itemsimilarity takes a text file that is expected to have user and item IDs that conform to +Mahout's mapreduce version of itemsimilarity takes a text file that is expected to have user and item IDs that conform to Mahout's ID requirements--they are non-negative integers that can be viewed as row and column numbers in a matrix. -*spark-itemsimilarity* also extends the notion of cooccurrence to cross-cooccurrence, in other words the Spark version will -account for multi-modal interactions and create cross-cooccurrence indicator matrices allowing the use of much more data in -creating recommendations or similar item lists. People try to do this by mixing different indicators and giving them weights. +*spark-itemsimilarity* also extends the notion of cooccurrence to cross-cooccurrence, in other words the Spark version will +account for multi-modal interactions and create cross-cooccurrence indicator matrices allowing the use of much more data in +creating recommendations or similar item lists. People try to do this by mixing different indicators and giving them weights. For instance they might say an item-view is 0.2 of an item purchase. In practice this is often not helpful. Spark-itemsimilarity's cross-cooccurrence is a more principled way to handle this case. In effect it scrubs secondary indicators with the indicator you want -to recommend. +to recommend. spark-itemsimilarity Mahout 1.0 Usage: spark-itemsimilarity [options] - + Disconnected from the target VM, address: '127.0.0.1:64676', transport: 'socket' Input, output options -i | --input @@ -55,15 +55,15 @@ to recommend. Secondary input path for cross-similarity calculation, same restrictions as "--input" (optional). Default: empty. -o | --output Path for output, any local or HDFS supported URI (required) - + Algorithm control options: -mppu | --maxPrefs Max number of preferences to consider per user (optional). Default: 500 -m | --maxSimilaritiesPerItem Limit the number of similarities per item to this number (optional). Default: 100 - + Note: Only the Log Likelihood Ratio (LLR) is supported as a similarity measure. - + Input text file schema options: -id | --inDelim Input delimiter character (optional). Default: "[,\t]" @@ -77,15 +77,15 @@ to recommend. Column number (0 based Int) containing the item ID string (optional). Default: 1 -fc | --filterColumn Column number (0 based Int) containing the filter string (optional). Default: -1 for no filter - + Using all defaults the input is expected of the form: "userIDitemId" or "userIDitemIDany-text..." and all rows will be used - + File discovery options: -r | --recursive Searched the -i path recursively for files that match --filenamePattern (optional), Default: false -fp | --filenamePattern Regex to match in determining input files (optional). Default: filename in the --input option or "^part-.*" if --input is a directory - + Output text file schema options: -rd | --rowKeyDelim Separates the rowID key from the vector values list (optional). Default: "\t" @@ -96,22 +96,22 @@ to recommend. -os | --omitStrength Do not write the strength to the output files (optional), Default: false. This option is used to output indexable data for creating a search engine recommender. - + Default delimiters will produce output of the form: "itemID1itemID2:value2itemID10:value10..." - + Spark config options: -ma | --master Spark Master URL (optional). Default: "local". Note that you can specify the number of cores to get a performance improvement, for example "local[4]" -sem | --sparkExecutorMem Max Java heap available as "executor memory" on each node (optional). Default: 4g -rs | --randomSeed - + -h | --help prints this usage text This looks daunting but defaults to simple fairly sane values to take exactly the same input as legacy code and is pretty flexible. It allows the user to point to a single text file, a directory full of files, or a tree of directories to be traversed recursively. The files included can be specified with either a regex-style pattern or filename. The schema for the file is defined by column numbers, which map to the important bits of data including IDs and values. The files can even contain filters, which allow unneeded rows to be discarded or used for cross-cooccurrence calculations. -See `ItemSimilarityDriver.scala` in Mahout's spark module if you want to customize the code. +See `ItemSimilarityDriver.scala` in Mahout's spark module if you want to customize the code. ### Defaults in the _**spark-itemsimilarity**_ CLI @@ -133,13 +133,13 @@ This will use the "local" Spark context and will output the standard text versio ### How To Use Multiple User Indicators -Often we record various indicators the user takes for later analytics. These can now be used to make recommendations. +Often we record various indicators the user takes for later analytics. These can now be used to make recommendations. The idea of a recommender is to recommend the action you want the user to make. For an ecom app this might be a purchase action recorded in a "purchase" indicator. It is usually not a good idea to just treat other indicators the same as the indicator you want to recommend. For example is you have user purchase and view data, never treat a view as a purchase it will never increase the quality of recommendations, instead use the view data as a secondary indicator so the CCO algorithm will find meaningful correlated cross-occurrences. Without this the views will be so noisy they will almost surely reduce the performance of the recommender. Too many people have fallen into this mistake. With *spark-itemsimilarity* we can now use both indicators. Mahout will use cross-occurrence analysis to limit the views to ones that do predict purchases. -We do this by treating the primary indicator (purchase) as data for the indicator matrix and use the secondary indicator (view) -to calculate the cross-cooccurrence indicator matrix. +We do this by treating the primary indicator (purchase) as data for the indicator matrix and use the secondary indicator (view) +to calculate the cross-cooccurrence indicator matrix. -*spark-itemsimilarity* can read separate indicators from separate files or from a mixed indicator log by filtering certain lines. For a mixed +*spark-itemsimilarity* can read separate indicators from separate files or from a mixed indicator log by filtering certain lines. For a mixed indicator log of the form: u1,purchase,iphone @@ -182,7 +182,7 @@ Use the following options: ### Output -The output of the job will be the standard text version of two Mahout DRMs. This is a case where we are calculating +The output of the job will be the standard text version of two Mahout DRMs. This is a case where we are calculating cross-cooccurrence so a primary indicator matrix and cross-cooccurrence indicator matrix will be created out-path @@ -230,7 +230,7 @@ A common method of storing data is in log files. If they are written using some 2014-06-23 14:46:53.115u3viewrandom textnexus 2014-06-23 14:46:53.115u4viewrandom textiphone 2014-06-23 14:46:53.115u4viewrandom textipad - 2014-06-23 14:46:53.115u4viewrandom textgalaxy + 2014-06-23 14:46:53.115u4viewrandom textgalaxy Can be parsed with the following CLI and run on the cluster producing the same output as the above example. The important bit of information in the example tab delimited file are user-id, indicator-name, and item-id. The rest is ignored. @@ -247,35 +247,35 @@ Can be parsed with the following CLI and run on the cluster producing the same o ## 2. spark-rowsimilarity -*spark-rowsimilarity* is the companion to *spark-itemsimilarity* the primary difference is that it takes a text file version of +*spark-rowsimilarity* is the companion to *spark-itemsimilarity* the primary difference is that it takes a text file version of a matrix of sparse vectors with optional application specific IDs and it finds similar rows rather than items (columns). Its use is -not limited to collaborative filtering. The input is in text-delimited form where there are three delimiters used. By +not limited to collaborative filtering. The input is in text-delimited form where there are three delimiters used. By default it reads `(rowIDcolumnID1:strength1columnID2:strength2...)` Since this job only supports LLR similarity, - which does not use the input strengths, they may be omitted in the input. It writes -`(rowIDrowID1:strength1rowID2:strength2...)` -The output is sorted by strength descending. The output can be interpreted as a row ID from the primary input followed + which does not use the input strengths, they may be omitted in the input. It writes +`(rowIDrowID1:strength1rowID2:strength2...)` +The output is sorted by strength descending. The output can be interpreted as a row ID from the primary input followed by a list of the most similar rows. The command line interface is: spark-rowsimilarity Mahout 0.x Usage: spark-rowsimilarity [options] - + Input, output options -i | --input Input path, may be a filename, directory name, or comma delimited list of HDFS supported URIs (required) -o | --output Path for output, any local or HDFS supported URI (required) - + Algorithm control options: -mo | --maxObservations Max number of observations to consider per row (optional). Default: 500 -m | --maxSimilaritiesPerRow Limit the number of similarities per item to this number (optional). Default: 100 - + Note: Only the Log Likelihood Ratio (LLR) is supported as a similarity measure. Disconnected from the target VM, address: '127.0.0.1:49162', transport: 'socket' - + Output text file schema options: -rd | --rowKeyDelim Separates the rowID key from the vector values list (optional). Default: "\t" @@ -286,39 +286,39 @@ The command line interface is: -os | --omitStrength Do not write the strength to the output files (optional), Default: false. This option is used to output indexable data for creating a search engine recommender. - + Default delimiters will produce output of the form: "itemID1itemID2:value2itemID10:value10..." - + File discovery options: -r | --recursive Searched the -i path recursively for files that match --filenamePattern (optional), Default: false -fp | --filenamePattern Regex to match in determining input files (optional). Default: filename in the --input option or "^part-.*" if --input is a directory - + Spark config options: -ma | --master Spark Master URL (optional). Default: "local". Note that you can specify the number of cores to get a performance improvement, for example "local[4]" -sem | --sparkExecutorMem Max Java heap available as "executor memory" on each node (optional). Default: 4g -rs | --randomSeed - + -h | --help prints this usage text -See RowSimilarityDriver.scala in Mahout's spark module if you want to customize the code. +See RowSimilarityDriver.scala in Mahout's spark module if you want to customize the code. #3. Using *spark-rowsimilarity* with Text Data -Another use case for *spark-rowsimilarity* is in finding similar textual content. For instance given the tags associated with -a blog post, which other posts have similar tags. In this case the columns are tags and the rows are posts. Since LLR is -the only similarity method supported this is not the optimal way to determine general "bag-of-words" document similarity. -LLR is used more as a quality filter than as a similarity measure. However *spark-rowsimilarity* will produce +Another use case for *spark-rowsimilarity* is in finding similar textual content. For instance given the tags associated with +a blog post, which other posts have similar tags. In this case the columns are tags and the rows are posts. Since LLR is +the only similarity method supported this is not the optimal way to determine general "bag-of-words" document similarity. +LLR is used more as a quality filter than as a similarity measure. However *spark-rowsimilarity* will produce lists of similar docs for every doc if input is docs with lists of terms. The Apache [Lucene](http://lucene.apache.org) project provides several methods of analyzing and tokenizing documents. # 4. Creating a Multimodal Recommender Using the output of *spark-itemsimilarity* and *spark-rowsimilarity* you can build a miltimodal cooccurrence and content based - recommender that can be used in both or either mode depending on indicators available and the history available at + recommender that can be used in both or either mode depending on indicators available and the history available at runtime for a user. Some slide describing this method can be found [here](http://occamsmachete.com/ml/2014/10/07/creating-a-unified-recommender-with-mahout-and-a-search-engine/) ## Requirements @@ -336,43 +336,43 @@ Indicators come in 3 types 2. **Content**: calculated from item metadata or content using *spark-rowsimilarity* 3. **Intrinsic**: assigned to items as metadata. Can be anything that describes the item. These will be used in search engine queries to implement business rules. -The query for recommendations will be a mix of values meant to match one of your indicators. The query can be constructed -from user history and values derived from context (category being viewed for instance) or special pre-calculated data -(popularity rank for instance). This blending of indicators allows for creating many flavors or recommendations to fit +The query for recommendations will be a mix of values meant to match one of your indicators. The query can be constructed +from user history and values derived from context (category being viewed for instance) or special pre-calculated data +(popularity rank for instance). This blending of indicators allows for creating many flavors or recommendations to fit a very wide variety of circumstances. -With the right mix of indicators developers can construct a single query that works for completely new items and new users +With the right mix of indicators developers can construct a single query that works for completely new items and new users while working well for items with lots of interactions and users with many recorded indicators. In other words by adding in content and intrinsic indicators developers can create a solution for the "cold-start" problem that gracefully improves with more user history -and as items have more interactions. It is also possible to create a completely content-based recommender that personalizes +and as items have more interactions. It is also possible to create a completely content-based recommender that personalizes recommendations. ## Example with 3 Indicators -You will need to decide how you store user indicator data so they can be processed by the item and row similarity jobs and -this is most easily done by using text files as described above. The data that is processed by these jobs is considered the -training data. You will need some amount of user history in your recs query. It is typical to use the most recent user history -but need not be exactly what is in the training set, which may include a greater volume of historical data. Keeping the user -history for query purposes could be done with a database by storing it in a users table. In the example above the two -collaborative filtering indicators are "purchase" and "view", but let's also add tags (taken from catalog categories or other -descriptive metadata). +You will need to decide how you store user indicator data so they can be processed by the item and row similarity jobs and +this is most easily done by using text files as described above. The data that is processed by these jobs is considered the +training data. You will need some amount of user history in your recs query. It is typical to use the most recent user history +but need not be exactly what is in the training set, which may include a greater volume of historical data. Keeping the user +history for query purposes could be done with a database by storing it in a users table. In the example above the two +collaborative filtering indicators are "purchase" and "view", but let's also add tags (taken from catalog categories or other +descriptive metadata). -We will need to create 1 cooccurrence indicator from the primary indicator (purchase) 1 cross-occurrence indicator -from the secondary indicator (view) +We will need to create 1 cooccurrence indicator from the primary indicator (purchase) 1 cross-occurrence indicator +from the secondary indicator (view) and 1 content indicator (tags). We'll have to run *spark-itemsimilarity* once and *spark-rowsimilarity* once. -We have described how to create the collaborative filtering indicators for purchase and view (the [How to use Multiple User -Indicators](#multiple-actions) section) but tags will be a slightly different process. We want to use the fact that -certain items have tags similar to the ones associated with a user's purchases. This is not a collaborative filtering indicator -but rather a "content" or "metadata" type indicator since you are not using other users' history, only the -individual that you are making recs for. This means that this method will make recommendations for items that have +We have described how to create the collaborative filtering indicators for purchase and view (the [How to use Multiple User +Indicators](#multiple-actions) section) but tags will be a slightly different process. We want to use the fact that +certain items have tags similar to the ones associated with a user's purchases. This is not a collaborative filtering indicator +but rather a "content" or "metadata" type indicator since you are not using other users' history, only the +individual that you are making recs for. This means that this method will make recommendations for items that have no collaborative filtering data, as happens with new items in a catalog. New items may have tags assigned but no one has purchased or viewed them yet. In the final query we will mix all 3 indicators. ## Content Indicator -To create a content-indicator we'll make use of the fact that the user has purchased items with certain tags. We want to find -items with the most similar tags. Notice that other users' behavior is not considered--only other item's tags. This defines a -content or metadata indicator. They are used when you want to find items that are similar to other items by using their +To create a content-indicator we'll make use of the fact that the user has purchased items with certain tags. We want to find +items with the most similar tags. Notice that other users' behavior is not considered--only other item's tags. This defines a +content or metadata indicator. They are used when you want to find items that are similar to other items by using their content or metadata, not by which users interacted with them. **Note**: It may be advisable to treat tags as cross-cooccurrence indicators but for the sake of an example they are treated here as content only. @@ -388,9 +388,9 @@ The full collection will look like the tags column from a catalog DB. For our ec 9446577dwomen tops chambray clothing casual ... -We'll use *spark-rowimilairity* because we are looking for similar rows, which encode items in this case. As with the -collaborative filtering indicators we use the --omitStrength option. The strengths created are -probabilistic log-likelihood ratios and so are used to filter unimportant similarities. Once the filtering or downsampling +We'll use *spark-rowimilairity* because we are looking for similar rows, which encode items in this case. As with the +collaborative filtering indicators we use the --omitStrength option. The strengths created are +probabilistic log-likelihood ratios and so are used to filter unimportant similarities. Once the filtering or downsampling is finished we no longer need the strengths. We will get an indicator matrix of the form: itemIDlist-of-item IDs @@ -400,30 +400,30 @@ This is a content indicator since it has found other items with similar content 3459860b3459860b 3459860b 6749860c 5959860a 3434860a 3477860a 9446577d9446577d 9496577d 0943577d 8346577d 9442277d 9446577e - ... - + ... + We now have three indicators, two collaborative filtering type and one content type. ## Multimodal Recommender Query -The actual form of the query for recommendations will vary depending on your search engine but the intent is the same. For a given user, map their history of an indicator or content to the correct indicator field and perform an OR'd query. +The actual form of the query for recommendations will vary depending on your search engine but the intent is the same. For a given user, map their history of an indicator or content to the correct indicator field and perform an OR'd query. -We have 3 indicators, these are indexed by the search engine into 3 fields, we'll call them "purchase", "view", and "tags". +We have 3 indicators, these are indexed by the search engine into 3 fields, we'll call them "purchase", "view", and "tags". We take the user's history that corresponds to each indicator and create a query of the form: Query: field: purchase; q:user's-purchase-history field: view; q:user's view-history field: tags; q:user's-tags-associated-with-purchases - -The query will result in an ordered list of items recommended for purchase but skewed towards items with similar tags to -the ones the user has already purchased. -This is only an example and not necessarily the optimal way to create recs. It illustrates how business rules can be -translated into recommendations. This technique can be used to skew recommendations towards intrinsic indicators also. -For instance you may want to put personalized popular item recs in a special place in the UI. Create a popularity indicator +The query will result in an ordered list of items recommended for purchase but skewed towards items with similar tags to +the ones the user has already purchased. + +This is only an example and not necessarily the optimal way to create recs. It illustrates how business rules can be +translated into recommendations. This technique can be used to skew recommendations towards intrinsic indicators also. +For instance you may want to put personalized popular item recs in a special place in the UI. Create a popularity indicator by tagging items with some category of popularity (hot, warm, cold for instance) then -index that as a new indicator field and include the corresponding value in a query +index that as a new indicator field and include the corresponding value in a query on the popularity field. If we use the ecom example but use the query to get "hot" recommendations it might look like this: Query: diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/recommenders/d-als.md b/website/Map_Reduce_Folder/docs/latest/algorithms/recommenders/d-als.md index 5ead0828c8..0c87049a4d 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/recommenders/d-als.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/recommenders/d-als.md @@ -6,16 +6,16 @@ redirect_from: - /docs/latest/algorithms/recomenders/d-als.html --- -Seems like someone has jacked up this page? +Seems like someone has jacked up this page? TODO: Find the ALS Page ## Intro Mahout has a distributed implementation of QR decomposition for tall thin matricies[1]. -## Algorithm +## Algorithm -For the classic QR decomposition of the form `\(\mathbf{A}=\mathbf{QR},\mathbf{A}\in\mathbb{R}^{m\times n}\)` a distributed version is fairly easily achieved if `\(\mathbf{A}\)` is tall and thin such that `\(\mathbf{A}^{\top}\mathbf{A}\)` fits in memory, i.e. *m* is large but *n* < ~5000 Under such circumstances, only `\(\mathbf{A}\)` and `\(\mathbf{Q}\)` are distributed matricies and `\(\mathbf{A^{\top}A}\)` and `\(\mathbf{R}\)` are in-core products. We just compute the in-core version of the Cholesky decomposition in the form of `\(\mathbf{LL}^{\top}= \mathbf{A}^{\top}\mathbf{A}\)`. After that we take `\(\mathbf{R}= \mathbf{L}^{\top}\)` and `\(\mathbf{Q}=\mathbf{A}\left(\mathbf{L}^{\top}\right)^{-1}\)`. The latter is easily achieved by multiplying each verticle block of `\(\mathbf{A}\)` by `\(\left(\mathbf{L}^{\top}\right)^{-1}\)`. (There is no actual matrix inversion happening). +For the classic QR decomposition of the form `\(\mathbf{A}=\mathbf{QR},\mathbf{A}\in\mathbb{R}^{m\times n}\)` a distributed version is fairly easily achieved if `\(\mathbf{A}\)` is tall and thin such that `\(\mathbf{A}^{\top}\mathbf{A}\)` fits in memory, i.e. *m* is large but *n* < ~5000 Under such circumstances, only `\(\mathbf{A}\)` and `\(\mathbf{Q}\)` are distributed matricies and `\(\mathbf{A^{\top}A}\)` and `\(\mathbf{R}\)` are in-core products. We just compute the in-core version of the Cholesky decomposition in the form of `\(\mathbf{LL}^{\top}= \mathbf{A}^{\top}\mathbf{A}\)`. After that we take `\(\mathbf{R}= \mathbf{L}^{\top}\)` and `\(\mathbf{Q}=\mathbf{A}\left(\mathbf{L}^{\top}\right)^{-1}\)`. The latter is easily achieved by multiplying each verticle block of `\(\mathbf{A}\)` by `\(\left(\mathbf{L}^{\top}\right)^{-1}\)`. (There is no actual matrix inversion happening). @@ -23,7 +23,7 @@ For the classic QR decomposition of the form `\(\mathbf{A}=\mathbf{QR},\mathbf{A Mahout `dqrThin(...)` is implemented in the mahout `math-scala` algebraic optimizer which translates Mahout's R-like linear algebra operators into a physical plan for both Spark and H2O distributed engines. - def dqrThin[K: ClassTag](A: DrmLike[K], checkRankDeficiency: Boolean = true): (DrmLike[K], Matrix) = { + def dqrThin[K: ClassTag](A: DrmLike[K], checkRankDeficiency: Boolean = true): (DrmLike[K], Matrix) = { if (drmA.ncol > 5000) log.warn("A is too fat. A'A must fit in memory and easily broadcasted.") implicit val ctx = drmA.context @@ -48,13 +48,12 @@ The scala `dqrThin(...)` method can easily be called in any Spark or H2O applica import org.apache.mahout.math._ import decompositions._ import drm._ - + val(drmQ, inCoreR) = dqrThin(drma) - + ## References [1]: [Mahout Scala and Mahout Spark Bindings for Linear Algebra Subroutines](http://mahout.apache.org/users/sparkbindings/ScalaSparkBindings.pdf) [2]: [Mahout Spark and Scala Bindings](http://mahout.apache.org/users/sparkbindings/home.html) - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/recommenders/index.md b/website/Map_Reduce_Folder/docs/latest/algorithms/recommenders/index.md index cae05d05a4..2b312b56a3 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/recommenders/index.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/recommenders/index.md @@ -23,7 +23,7 @@ To get the most out of our more modern CCO algorithm we'll need to think of the To integrate with your application you will collect user interactions storing them in a DB and also in a from usable by Mahout. The simplest way to do this is to log user interactions to csv files (user-id, item-id). The DB should be setup to contain the last n user interactions, which will form part of the query for recommendations. -Mahout's spark-itemsimilarity will create a table of (item-id, list-of-similar-items) in csv form. Think of this as an item collection with one field containing the item-ids of similar items. Index this with your search engine. +Mahout's spark-itemsimilarity will create a table of (item-id, list-of-similar-items) in csv form. Think of this as an item collection with one field containing the item-ids of similar items. Index this with your search engine. When your application needs recommendations for a specific person, get the latest user history of interactions from the DB and query the indicator collection with this history. You will get back an ordered list of item-ids. These are your recommendations. You may wish to filter out any that the user has already seen but that will depend on your use case. diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/regression/fittness-tests.md b/website/Map_Reduce_Folder/docs/latest/algorithms/regression/fittness-tests.md index ff53a6cd21..b91d8856e5 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/regression/fittness-tests.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/regression/fittness-tests.md @@ -14,5 +14,3 @@ Stub ### Parameters ### Example - - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/regression/ols.md b/website/Map_Reduce_Folder/docs/latest/algorithms/regression/ols.md index 1cca7a5e28..2acf739278 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/regression/ols.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/regression/ols.md @@ -8,10 +8,10 @@ redirect_from: ### About -The `OrinaryLeastSquares` regressor in Mahout implements a _closed-form_ solution to [Ordinary Least Squares](https://en.wikipedia.org/wiki/Ordinary_least_squares). +The `OrinaryLeastSquares` regressor in Mahout implements a _closed-form_ solution to [Ordinary Least Squares](https://en.wikipedia.org/wiki/Ordinary_least_squares). This is in stark contrast to many "big data machine learning" frameworks which implement a _stochastic_ approach. From the users perspecive this difference can be reduced to: -- **_Stochastic_**- A series of guesses at a line line of best fit. +- **_Stochastic_**- A series of guesses at a line line of best fit. - **_Closed Form_**- A mathimatical approach has been explored, the properties of the parameters are well understood, and problems which arise (and the remedial measures), exist. This is usually the preferred choice of mathematicians/statisticians, but computational limititaions have forced us to resort to SGD. ### Parameters @@ -37,7 +37,7 @@ This is in stark contrast to many "big data machine learning" frameworks which i - +
'addIntercept Add an intercept to \(\mathbf{X}\) true
diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/regression/ridge.md b/website/Map_Reduce_Folder/docs/latest/algorithms/regression/ridge.md index 2ced86a8ca..c72047e585 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/regression/ridge.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/regression/ridge.md @@ -42,7 +42,7 @@ by using a Ridge regression, penalizing higher values of beta coefficients. 'addIntercept Add an intercept to \(\mathbf{X}\) true - + diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/regression/serial-correlation/cochrane-orcutt.md b/website/Map_Reduce_Folder/docs/latest/algorithms/regression/serial-correlation/cochrane-orcutt.md index cee1e70fef..5ca07c901b 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/regression/serial-correlation/cochrane-orcutt.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/regression/serial-correlation/cochrane-orcutt.md @@ -8,11 +8,11 @@ redirect_from: ### About -The [Cochrane Orcutt](https://en.wikipedia.org/wiki/Cochrane%E2%80%93Orcutt_estimation) procedure is use in economics to -adjust a linear model for serial correlation in the error term. - +The [Cochrane Orcutt](https://en.wikipedia.org/wiki/Cochrane%E2%80%93Orcutt_estimation) procedure is use in economics to +adjust a linear model for serial correlation in the error term. + The cooresponding method in R is [`cochrane.orcutt`](https://cran.r-project.org/web/packages/orcutt/orcutt.pdf) -however the implementation differes slightly. +however the implementation differes slightly. #### R Prototype: library(orcutt) @@ -44,7 +44,7 @@ however the implementation differes slightly. my_lm = lm(y ~ x, data=df) coch = cochrane.orcutt(my_lm) - + The R-implementation is kind of...silly. The above works- converges at 318 iterations- the transformed DW is 1.72, yet the rho is @@ -52,8 +52,8 @@ The above works- converges at 318 iterations- the transformed DW is 1.72, yet autocorrelation- nothing close to 1.72. At anyrate, the real prototype for this is the example from [Applied Linear Statistcal Models - 5th Edition by Kunter, Nachstheim, Neter, and Li](https://www.amazon.com/Applied-Linear-Statistical-Models-Hardcover/dp/B010EWX85C/ref=sr_1_4?ie=UTF8&qid=1493847480&sr=8-4&keywords=applied+linear+statistical+models+5th+edition). - + 5th Edition by Kunter, Nachstheim, Neter, and Li](https://www.amazon.com/Applied-Linear-Statistical-Models-Hardcover/dp/B010EWX85C/ref=sr_1_4?ie=UTF8&qid=1493847480&sr=8-4&keywords=applied+linear+statistical+models+5th+edition). + Steps: 1. Normal Regression 2. Estimate \(\rho\) @@ -61,10 +61,10 @@ Steps: 4. Step 5: Use Betas from (4) to recalculate model from (1) 5. Step 6: repeat Step 2 through 5 until a stopping criteria is met. Some models call for convergence- Kunter et. al reccomend 3 iterations, if you don't achieve desired results, use an alternative method. - + #### Some additional notes from Applied Linear Statistical Models: They also provide some interesting notes on p 494: - + 1. "Cochrane-Orcutt does not always work properly. A major reason is that when the error terms are positively autocorrelated, the estimate \(r\) in (12.22) tends to underestimate the autocorrelation parameter \(\rho\). When this bias is serious, it can significantly reduce the effectiveness of the @@ -108,7 +108,7 @@ Kunter et. al reccomend 3 iterations, if you don't achieve desired results, use 'cacheHint The DRM Cache Hint to use when holding the data in memory between iterations CacheHint.MEMORY_ONLY - + @@ -136,12 +136,11 @@ Kunter et. al reccomend 3 iterations, if you don't achieve desired results, use (27.78, 165.6), (28.24, 168.7), (28.78, 171.7) )) - + val drmY = alsmBlaisdellCo(::, 0 until 1) val drmX = alsmBlaisdellCo(::, 1 until 2) var coModel = new CochraneOrcutt[Int]().fit(drmX, drmY , ('iterations -> 2)) - + println(coModel.rhos) println(coModel.summary) - diff --git a/website/Map_Reduce_Folder/docs/latest/algorithms/template.md b/website/Map_Reduce_Folder/docs/latest/algorithms/template.md index 2c6e37f10b..7f05754136 100644 --- a/website/Map_Reduce_Folder/docs/latest/algorithms/template.md +++ b/website/Map_Reduce_Folder/docs/latest/algorithms/template.md @@ -14,8 +14,3 @@ Stub ### Parameters ### Example - - - - - diff --git a/website/Map_Reduce_Folder/docs/latest/distributed/flink-bindings.md b/website/Map_Reduce_Folder/docs/latest/distributed/flink-bindings.md index 110d6335bb..e0313b5b3c 100644 --- a/website/Map_Reduce_Folder/docs/latest/distributed/flink-bindings.md +++ b/website/Map_Reduce_Folder/docs/latest/distributed/flink-bindings.md @@ -17,7 +17,7 @@ The Mahout DRM, or Distributed Row Matrix, is an abstraction for storing a large # Flink Overview -Apache Flink is an open source, distributed Stream and Batch Processing Framework. At it's core, Flink is a Stream Processing engine and Batch processing is an extension of Stream Processing. +Apache Flink is an open source, distributed Stream and Batch Processing Framework. At it's core, Flink is a Stream Processing engine and Batch processing is an extension of Stream Processing. Flink includes several APIs for building applications with the Flink Engine: @@ -46,5 +46,3 @@ flink/src/main/scala/org/apache/mahout/flinkbindings/drm/*.scala - Flink Dataset flink/src/main/scala/org/apache/mahout/flinkbindings/io/*.scala - Read / Write between DRMDataSet and files on HDFS flink/src/main/scala/org/apache/mahout/flinkbindings/FlinkEngine.scala - DSL operator graph evaluator and various abstract API implementations for a distributed engine. - - diff --git a/website/Map_Reduce_Folder/docs/latest/distributed/h2o-internals.md b/website/Map_Reduce_Folder/docs/latest/distributed/h2o-internals.md index 6486927469..be39222d9e 100644 --- a/website/Map_Reduce_Folder/docs/latest/distributed/h2o-internals.md +++ b/website/Map_Reduce_Folder/docs/latest/distributed/h2o-internals.md @@ -6,7 +6,7 @@ redirect_from: - /docs/latest/distributed/h2o-internals.html --- # Introduction - + This document provides an overview of how the Mahout Samsara environment is implemented over the H2O backend engine. The document is aimed at Mahout developers, to give a high level description of the design so that one can explore the code inside `h2o/` with some context. ## H2O Overview diff --git a/website/Map_Reduce_Folder/docs/latest/distributed/spark-bindings/faq.md b/website/Map_Reduce_Folder/docs/latest/distributed/spark-bindings/faq.md index 8cd2a2276d..4419b51201 100644 --- a/website/Map_Reduce_Folder/docs/latest/distributed/spark-bindings/faq.md +++ b/website/Map_Reduce_Folder/docs/latest/distributed/spark-bindings/faq.md @@ -4,51 +4,47 @@ title: FAQ redirect_from: - /docs/latest/distributed/spark-bindings/faq - /docs/latest/distributed/spark-bindings/faq.html - + --- # FAQ for using Mahout with Spark **Q: Mahout Spark shell doesn't start; "ClassNotFound" problems or various classpath problems.** -**A:** So far as of the time of this writing all reported problems starting the Spark shell in Mahout were revolving -around classpath issues one way or another. +**A:** So far as of the time of this writing all reported problems starting the Spark shell in Mahout were revolving +around classpath issues one way or another. -If you are getting method signature like errors, most probably you have mismatch between Mahout's Spark dependency +If you are getting method signature like errors, most probably you have mismatch between Mahout's Spark dependency and actual Spark installed. (At the time of this writing the HEAD depends on Spark 2.1.0) but check mahout/pom.xml. -Troubleshooting general classpath issues is pretty straightforward. Since Mahout is using Spark's installation -and its classpath as reported by Spark itself for Spark-related dependencies, it is important to make sure +Troubleshooting general classpath issues is pretty straightforward. Since Mahout is using Spark's installation +and its classpath as reported by Spark itself for Spark-related dependencies, it is important to make sure the classpath is sane and is made available to Mahout: 1. Check Spark is of correct version (same as in Mahout's poms), is compiled and SPARK_HOME is set. 2. Check Mahout is compiled and MAHOUT_HOME is set. -3. Run `$SPARK_HOME/bin/compute-classpath.sh` and make sure it produces sane result with no errors. -If it outputs something other than a straightforward classpath string, most likely Spark is not compiled/set correctly (later spark versions require +3. Run `$SPARK_HOME/bin/compute-classpath.sh` and make sure it produces sane result with no errors. +If it outputs something other than a straightforward classpath string, most likely Spark is not compiled/set correctly (later spark versions require `sbt/sbt assembly` to be run, simply runnig `sbt/sbt publish-local` is not enough any longer). 4. Run `$MAHOUT_HOME/bin/mahout -spark classpath` and check that path reported in step (3) is included. -**Q: I am using the command line Mahout jobs that run on Spark or am writing my own application that uses -Mahout's Spark code. When I run the code on my cluster I get ClassNotFound or signature errors during serialization. +**Q: I am using the command line Mahout jobs that run on Spark or am writing my own application that uses +Mahout's Spark code. When I run the code on my cluster I get ClassNotFound or signature errors during serialization. What's wrong?** - -**A:** The Spark artifacts in the maven ecosystem may not match the exact binary you are running on your cluster. This may -cause class name or version mismatches. In this case you may wish + +**A:** The Spark artifacts in the maven ecosystem may not match the exact binary you are running on your cluster. This may +cause class name or version mismatches. In this case you may wish to build Spark yourself to guarantee that you are running exactly what you are building Mahout against. To do this follow these steps in order: -1. Build Spark with maven, but **do not** use the "package" target as described on the Spark site. Build with the "clean install" target instead. +1. Build Spark with maven, but **do not** use the "package" target as described on the Spark site. Build with the "clean install" target instead. Something like: "mvn clean install -Dhadoop1.2.1" or whatever your particular build options are. This will put the jars for Spark in the local maven cache. 2. Deploy **your** Spark build to your cluster and test it there. -3. Build Mahout. This will cause maven to pull the jars for Spark from the local maven cache and may resolve missing +3. Build Mahout. This will cause maven to pull the jars for Spark from the local maven cache and may resolve missing or mis-identified classes. 4. if you are building your own code do so against the local builds of Spark and Mahout. **Q: The implicit SparkContext 'sc' does not work in the Mahout spark-shell.** -**A:** In the Mahout spark-shell the SparkContext is called 'sdc', where the 'd' stands for distributed. - - - - +**A:** In the Mahout spark-shell the SparkContext is called 'sdc', where the 'd' stands for distributed. diff --git a/website/Map_Reduce_Folder/docs/latest/distributed/spark-bindings/index.md b/website/Map_Reduce_Folder/docs/latest/distributed/spark-bindings/index.md index fbcd3055c9..93fc12214d 100644 --- a/website/Map_Reduce_Folder/docs/latest/distributed/spark-bindings/index.md +++ b/website/Map_Reduce_Folder/docs/latest/distributed/spark-bindings/index.md @@ -12,7 +12,7 @@ redirect_from: ## What is Scala & Spark Bindings? In short, Scala & Spark Bindings for Mahout is Scala DSL and algebraic optimizer of something like this (actual formula from **(d)spca**) - + `\[\mathbf{G}=\mathbf{B}\mathbf{B}^{\top}-\mathbf{C}-\mathbf{C}^{\top}+\mathbf{s}_{q}\mathbf{s}_{q}^{\top}\boldsymbol{\xi}^{\top}\boldsymbol{\xi}\]` @@ -23,30 +23,30 @@ Mahout Scala & Spark Bindings expression of the above: val g = bt.t %*% bt - c - c.t + (s_q cross s_q) * (xi dot xi) -The main idea is that a scientist writing algebraic expressions cannot care less of distributed +The main idea is that a scientist writing algebraic expressions cannot care less of distributed operation plans and works **entirely on the logical level** just like he or she would do with R. -Another idea is decoupling logical expression from distributed back-end. As more back-ends are added, +Another idea is decoupling logical expression from distributed back-end. As more back-ends are added, this implies **"write once, run everywhere"**. The linear algebra side works with scalars, in-core vectors and matrices, and Mahout Distributed Row Matrices (DRMs). -The ecosystem of operators is built in the R's image, i.e. it follows R naming such as %*%, -colSums, nrow, length operating over vectors or matices. +The ecosystem of operators is built in the R's image, i.e. it follows R naming such as %*%, +colSums, nrow, length operating over vectors or matices. -Important part of Spark Bindings is expression optimizer. It looks at expression as a whole +Important part of Spark Bindings is expression optimizer. It looks at expression as a whole and figures out how it can be simplified, and which physical operators should be picked. For example, there are currently about 5 different physical operators performing DRM-DRM multiplication -picked based on matrix geometry, distributed dataset partitioning, orientation etc. -If we count in DRM by in-core combinations, that would be another 4, i.e. 9 total -- all of it for just +picked based on matrix geometry, distributed dataset partitioning, orientation etc. +If we count in DRM by in-core combinations, that would be another 4, i.e. 9 total -- all of it for just simple x %*% y logical notation. Please refer to the documentation for details. ## Status -This environment addresses mostly R-like Linear Algebra optmizations for +This environment addresses mostly R-like Linear Algebra optmizations for Spark, Flink and H20. @@ -60,7 +60,7 @@ Spark, Flink and H20. * In-core ([ssvd]) and Distributed ([dssvd]) Stochastic SVD -- guinea pigs -- see the bindings manual * In-core ([spca]) and Distributed ([dspca]) Stochastic PCA -- guinea pigs -- see the bindings manual -* Distributed thin QR decomposition ([dqrThin]) -- guinea pig -- see the bindings manual +* Distributed thin QR decomposition ([dqrThin]) -- guinea pig -- see the bindings manual * [Current list of algorithms](https://mahout.apache.org/users/basics/algorithms.html) [ssvd]: https://github.com/apache/mahout/blob/trunk/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/SSVD.scala @@ -74,7 +74,7 @@ TODO TODO: Do we still want this? (I don't think so...) -## Related history of note +## Related history of note * CLI and Driver for Spark version of item similarity -- [MAHOUT-1541](https://issues.apache.org/jira/browse/MAHOUT-1541) * Command line interface for generalizable Spark pipelines -- [MAHOUT-1569](https://issues.apache.org/jira/browse/MAHOUT-1569) @@ -86,19 +86,16 @@ TODO: Do we still want this? (I don't think so...) * Full abstraction of DRM apis and algorithms from a distributed engine -- [MAHOUT-1529](https://issues.apache.org/jira/browse/MAHOUT-1529) * Port Naive Bayes -- [MAHOUT-1493](https://issues.apache.org/jira/browse/MAHOUT-1493) -## Work in progress +## Work in progress * Text-delimited files for input and output -- [MAHOUT-1568](https://issues.apache.org/jira/browse/MAHOUT-1568) * *Your issue here!* - - - - diff --git a/website/Map_Reduce_Folder/docs/latest/index.md b/website/Map_Reduce_Folder/docs/latest/index.md index e7ee00c2a5..828ff3d0d4 100755 --- a/website/Map_Reduce_Folder/docs/latest/index.md +++ b/website/Map_Reduce_Folder/docs/latest/index.md @@ -16,49 +16,49 @@ to start. Let's provide an overview to help you see how the pieces fit together 1. Logical/Physical DAG 1. Engine Bindings 1. Code runs in Engine -1. Native Solvers +1. Native Solvers ## Application Code You have an JAVA/Scala applicatoin (skip this if you're working from an interactive shell or Apache Zeppelin) - + def main(args: Array[String]) { println("Welcome to My Mahout App") if (args.isEmpty) { -This may seem like a trivial part to call out, but the point is important- Mahout runs _inline_ with your regular application -code. E.g. if this is an Apache Spark app, then you do all your Spark things, including ETL and data prep in the same +This may seem like a trivial part to call out, but the point is important- Mahout runs _inline_ with your regular application +code. E.g. if this is an Apache Spark app, then you do all your Spark things, including ETL and data prep in the same application, and then invoke Mahout's mathematically expressive Scala DSL when you're ready to math on it. ## Samsara Scala-DSL (Syntactic Sugar) -So when you get to a point in your code where you're ready to math it up (in this example Spark) you can elegantly express +So when you get to a point in your code where you're ready to math it up (in this example Spark) you can elegantly express yourself mathematically. implicit val sdc: org.apache.mahout.sparkbindings.SparkDistributedContext = sc2sdc(sc) - + val A = drmWrap(rddA) - val B = drmWrap(rddB) - + val B = drmWrap(rddB) + val C = A.t %*% A + A %*% B.t - + We've defined a `MahoutDistributedContext` (which is a wrapper on the Spark Context), and two Distributed Row Matrices (DRMs) -which are wrappers around RDDs (in Spark). +which are wrappers around RDDs (in Spark). ## Logical / Physical DAG At this point there is a bit of optimization that happens. For example, consider the - + A.t %*% A - -Which is + +Which is
\(\mathbf{A^\intercal A}\)
Transposing a large matrix is a very expensive thing to do, and in this case we don't actually need to do it: there is a -more efficient way to calculate \(\mathbf{A^\intercal A}\) that doesn't require a physical transpose. +more efficient way to calculate \(\mathbf{A^\intercal A}\) that doesn't require a physical transpose. (Image showing this) @@ -73,8 +73,8 @@ a series of operators, which are implemented at engine level_. When one creates new engine bindings, one is in essence defining: -1. What the engine specific underlying structure for a DRM is (in Spark its an RDD). The underlying structure also has -rows of `MahoutVector`s, so in Spark `RDD[(index, MahoutVector)]`. This will be important when we get to the native solvers. +1. What the engine specific underlying structure for a DRM is (in Spark its an RDD). The underlying structure also has +rows of `MahoutVector`s, so in Spark `RDD[(index, MahoutVector)]`. This will be important when we get to the native solvers. 1. Implementing a set of BLAS (basic linear algebra) functions for working on the underlying structure- in Spark this means implementing things like `AtA` on an RDD. See [the sparkbindings on github](https://github.com/apache/mahout/tree/master/spark/src/main/scala/org/apache/mahout/sparkbindings) @@ -82,31 +82,30 @@ Now your mathematically expressive Samsara Scala code has been translated into o ## Native Solvers -Recall how I said that rows of the DRMs are `org.apache.mahout.math.Vector`. Here is where this becomes important. I'm going -to explain this in the context of Spark, but the principals apply to all distributed backends. +Recall how I said that rows of the DRMs are `org.apache.mahout.math.Vector`. Here is where this becomes important. I'm going +to explain this in the context of Spark, but the principals apply to all distributed backends. -If you are familiar with how mapping and reducing in Spark, then envision this RDD of `MahoutVector`s, each partition, -and indexed collection of vectors is a _block_ of the distributed matrix, however this _block_ is totally in-core, and therefor is treated like an in-core matrix. +If you are familiar with how mapping and reducing in Spark, then envision this RDD of `MahoutVector`s, each partition, +and indexed collection of vectors is a _block_ of the distributed matrix, however this _block_ is totally in-core, and therefor is treated like an in-core matrix. Now Mahout defines its own in-core BLAS packs and refers to them as _Native Solvers_. The default native solver is just plain -old JVM, which is painfully slow, but works just about anywhere. +old JVM, which is painfully slow, but works just about anywhere. When the data gets to the node, an operation on the matrix block is called. In the same way Mahout converts abstract -operators on the DRM that are implemented on various distributed engines, it calls abstract operators on the in-core matrix -and vectors which are implemented on various native solvers. +operators on the DRM that are implemented on various distributed engines, it calls abstract operators on the in-core matrix +and vectors which are implemented on various native solvers. -The default "native solver" is the JVM, which isn't native at all, and if no actual native solvers are present operations +The default "native solver" is the JVM, which isn't native at all, and if no actual native solvers are present operations will fall back to this. However, IF a native solver is present (the jar was added to the notebook), then the magic will happen. Imagine still we have our Spark executor: it has this block of a matrix sitting in its core. Now let's suppose the `ViennaCl-OMP` -native solver is in use. When Spark calls an operation on this incore matrix, the matrix dumps out of the JVM and the -calculation is carried out on _all available CPUs_. +native solver is in use. When Spark calls an operation on this incore matrix, the matrix dumps out of the JVM and the +calculation is carried out on _all available CPUs_. In a similar way, the `ViennaCL` native solver dumps the matrix out of the JVM and looks for a GPU to execute the operations on. - -Once the operations are complete, the result is loaded back up into the JVM, and Spark (or whatever distributed engine) and -shipped back to the driver. -The native solver operations are only defined on `org.apache.mahout.math.Vector` and `org.apache.mahout.math.Matrix`, which is -why it is critical that the underlying structure is composed row-wise by `Vector` or `Matrices`. +Once the operations are complete, the result is loaded back up into the JVM, and Spark (or whatever distributed engine) and +shipped back to the driver. +The native solver operations are only defined on `org.apache.mahout.math.Vector` and `org.apache.mahout.math.Matrix`, which is +why it is critical that the underlying structure is composed row-wise by `Vector` or `Matrices`. diff --git a/website/Map_Reduce_Folder/docs/latest/mahout-samsara/faq.md b/website/Map_Reduce_Folder/docs/latest/mahout-samsara/faq.md index 4d8ee20ec9..a465cdf788 100644 --- a/website/Map_Reduce_Folder/docs/latest/mahout-samsara/faq.md +++ b/website/Map_Reduce_Folder/docs/latest/mahout-samsara/faq.md @@ -10,44 +10,40 @@ redirect_from: **Q: Mahout Spark shell doesn't start; "ClassNotFound" problems or various classpath problems.** -**A:** So far as of the time of this writing all reported problems starting the Spark shell in Mahout were revolving -around classpath issues one way or another. +**A:** So far as of the time of this writing all reported problems starting the Spark shell in Mahout were revolving +around classpath issues one way or another. -If you are getting method signature like errors, most probably you have mismatch between Mahout's Spark dependency +If you are getting method signature like errors, most probably you have mismatch between Mahout's Spark dependency and actual Spark installed. (At the time of this writing the HEAD depends on Spark 1.1.0) but check mahout/pom.xml. -Troubleshooting general classpath issues is pretty straightforward. Since Mahout is using Spark's installation -and its classpath as reported by Spark itself for Spark-related dependencies, it is important to make sure +Troubleshooting general classpath issues is pretty straightforward. Since Mahout is using Spark's installation +and its classpath as reported by Spark itself for Spark-related dependencies, it is important to make sure the classpath is sane and is made available to Mahout: 1. Check Spark is of correct version (same as in Mahout's poms), is compiled and SPARK_HOME is set. 2. Check Mahout is compiled and MAHOUT_HOME is set. -3. Run `$SPARK_HOME/bin/compute-classpath.sh` and make sure it produces sane result with no errors. -If it outputs something other than a straightforward classpath string, most likely Spark is not compiled/set correctly (later spark versions require +3. Run `$SPARK_HOME/bin/compute-classpath.sh` and make sure it produces sane result with no errors. +If it outputs something other than a straightforward classpath string, most likely Spark is not compiled/set correctly (later spark versions require `sbt/sbt assembly` to be run, simply runnig `sbt/sbt publish-local` is not enough any longer). 4. Run `$MAHOUT_HOME/bin/mahout -spark classpath` and check that path reported in step (3) is included. -**Q: I am using the command line Mahout jobs that run on Spark or am writing my own application that uses -Mahout's Spark code. When I run the code on my cluster I get ClassNotFound or signature errors during serialization. +**Q: I am using the command line Mahout jobs that run on Spark or am writing my own application that uses +Mahout's Spark code. When I run the code on my cluster I get ClassNotFound or signature errors during serialization. What's wrong?** - -**A:** The Spark artifacts in the maven ecosystem may not match the exact binary you are running on your cluster. This may -cause class name or version mismatches. In this case you may wish + +**A:** The Spark artifacts in the maven ecosystem may not match the exact binary you are running on your cluster. This may +cause class name or version mismatches. In this case you may wish to build Spark yourself to guarantee that you are running exactly what you are building Mahout against. To do this follow these steps in order: -1. Build Spark with maven, but **do not** use the "package" target as described on the Spark site. Build with the "clean install" target instead. +1. Build Spark with maven, but **do not** use the "package" target as described on the Spark site. Build with the "clean install" target instead. Something like: "mvn clean install -Dhadoop1.2.1" or whatever your particular build options are. This will put the jars for Spark in the local maven cache. 2. Deploy **your** Spark build to your cluster and test it there. -3. Build Mahout. This will cause maven to pull the jars for Spark from the local maven cache and may resolve missing +3. Build Mahout. This will cause maven to pull the jars for Spark from the local maven cache and may resolve missing or mis-identified classes. 4. if you are building your own code do so against the local builds of Spark and Mahout. **Q: The implicit SparkContext 'sc' does not work in the Mahout spark-shell.** -**A:** In the Mahout spark-shell the SparkContext is called 'sdc', where the 'd' stands for distributed. - - - - +**A:** In the Mahout spark-shell the SparkContext is called 'sdc', where the 'd' stands for distributed. diff --git a/website/Map_Reduce_Folder/docs/latest/mahout-samsara/in-core-reference.md b/website/Map_Reduce_Folder/docs/latest/mahout-samsara/in-core-reference.md index e376c47c05..4b8d4c09d6 100644 --- a/website/Map_Reduce_Folder/docs/latest/mahout-samsara/in-core-reference.md +++ b/website/Map_Reduce_Folder/docs/latest/mahout-samsara/in-core-reference.md @@ -15,7 +15,7 @@ The following imports are used to enable Mahout-Samsara's Scala DSL bindings for import org.apache.mahout.math._ import scalabindings._ import RLikeOps._ - + #### Inline initalization Dense vectors: @@ -30,13 +30,13 @@ Sparse vectors: // to create a vector with specific cardinality val sparseVec1 = svec((5 -> 1.0) :: (10 -> 2.0) :: Nil, cardinality = 20) - -Inline matrix initialization, either sparse or dense, is always done row wise. + +Inline matrix initialization, either sparse or dense, is always done row wise. Dense matrices: val A = dense((1, 2, 3), (3, 4, 5)) - + Sparse matrices: val A = sparse( @@ -51,11 +51,11 @@ Diagonal matrix with constant diagonal elements: Diagonal matrix with main diagonal backed by a vector: diagv((1, 2, 3, 4, 5)) - + Identity matrix: eye(10) - + ####Slicing and Assigning Getting a vector element: @@ -63,60 +63,60 @@ Getting a vector element: val d = vec(5) Setting a vector element: - + vec(5) = 3.0 - + Getting a matrix element: val d = m(3,5) - + Setting a matrix element: M(3,5) = 3.0 - + Getting a matrix row or column: val rowVec = M(3, ::) val colVec = M(::, 3) - + Setting a matrix row or column via vector assignment: M(3, ::) := (1, 2, 3) M(::, 3) := (1, 2, 3) - + Setting a subslices of a matrix row or column: a(0, 0 to 1) = (3, 5) - + Setting a subslices of a matrix row or column via vector assignment: a(0, 0 to 1) := (3, 5) - + Getting a matrix as from matrix contiguous block: val B = A(2 to 3, 3 to 4) - + Assigning a contiguous block to a matrix: A(0 to 1, 1 to 2) = dense((3, 2), (3 ,3)) - + Assigning a contiguous block to a matrix using the matrix assignment operator: A(o to 1, 1 to 2) := dense((3, 2), (3, 3)) - + Assignment operator used for copying between vectors or matrices: vec1 := vec2 M1 := M2 - + Assignment operator using assignment through a functional literal for a matrix: M := ((row, col, x) => if (row == col) 1 else 0 - + Assignment operator using assignment through a functional literal for a vector: vec := ((index, x) => sqrt(x) - + #### BLAS-like operations Plus/minus either vector or numeric with assignment or not: @@ -125,7 +125,7 @@ Plus/minus either vector or numeric with assignment or not: a - b a + 5.0 a - 5.0 - + Hadamard (elementwise) product, either vector or matrix or numeric operands: a * b @@ -139,41 +139,41 @@ Operations with assignment: a -= 5.0 a *= b a *= 5 - -*Some nuanced rules*: + +*Some nuanced rules*: 1/x in R (where x is a vector or a matrix) is elementwise inverse. In scala it would be expressed as: val xInv = 1 /: x and R's 5.0 - x would be: - + val x1 = 5.0 -: x - + *note: All assignment operations, including :=, return the assignee just like in C++*: - a -= b - -assigns **a - b** to **b** (in-place) and returns **b**. Similarly for **a /=: b** or **1 /=: v** - + a -= b + +assigns **a - b** to **b** (in-place) and returns **b**. Similarly for **a /=: b** or **1 /=: v** + Dot product: a dot b - + Matrix and vector equivalency (or non-equivalency). **Dangerous, exact equivalence is rarely useful, better to use norm comparisons with an allowance of small errors.** - + a === b a !== b - -Matrix multiply: + +Matrix multiply: a %*% b - -Optimized Right Multiply with a diagonal matrix: + +Optimized Right Multiply with a diagonal matrix: diag(5, 5) :%*% b - + Optimized Left Multiply with a diagonal matrix: A %*%: diag(5, 5) @@ -181,15 +181,15 @@ Optimized Left Multiply with a diagonal matrix: Second norm, of a vector or matrix: a.norm - + Transpose: val Mt = M.t - + *note: Transposition is currently handled via view, i.e. updating a transposed matrix will be updating the original.* Also computing something like `\(\mathbf{X^\top}\mathbf{X}\)`: val XtX = X.t %*% X - + will not therefore incur any additional data copying. #### Decompositions @@ -204,71 +204,71 @@ All arguments in the following are matricies. **Cholesky decomposition** val ch = chol(M) - + **SVD** val (U, V, s) = svd(M) - + **EigenDecomposition** val (V, d) = eigen(M) - + **QR decomposition** val (Q, R) = qr(M) - + **Rank**: Check for rank deficiency (runs rank-revealing QR) M.isFullRank - + **In-core SSVD** Val (U, V, s) = ssvd(A, k = 50, p = 15, q = 1) - + **Solving linear equation systems and matrix inversion:** fully similar to R semantics; there are three forms of invocation: Solve `\(\mathbf{AX}=\mathbf{B}\)`: solve(A, B) - + Solve `\(\mathbf{Ax}=\mathbf{b}\)`: - + solve(A, b) - + Compute `\(\mathbf{A^{-1}}\)`: solve(A) - + #### Misc Vector cardinality: a.length - + Matrix cardinality: m.nrow m.ncol - + Means and sums: m.colSums m.colMeans m.rowSums m.rowMeans - + Copy-By-Value: val b = a cloned - + #### Random Matrices `\(\mathcal{U}\)`(0,1) random matrix view: val incCoreA = Matrices.uniformView(m, n, seed) - + `\(\mathcal{U}\)`(-1,1) random matrix view: val incCoreA = Matrices.symmetricUniformView(m, n, seed) @@ -276,30 +276,28 @@ Copy-By-Value: `\(\mathcal{N}\)`(-1,1) random matrix view: val incCoreA = Matrices.gaussianView(m, n, seed) - -#### Iterators + +#### Iterators Mahout-Math already exposes a number of iterators. Scala code just needs the following imports to enable implicit conversions to scala iterators. import collection._ import JavaConversions._ - + Iterating over rows in a Matrix: for (row <- m) { ... do something with row } - + For more information including information on Mahout-Samsara's out-of-core Linear algebra bindings see: [Mahout Scala Bindings and Mahout Spark Bindings for Linear Algebra Subroutines](http://mahout.apache.org/users/sparkbindings/ScalaSparkBindings.pdf) - - diff --git a/website/Map_Reduce_Folder/docs/latest/mahout-samsara/out-of-core-reference.md b/website/Map_Reduce_Folder/docs/latest/mahout-samsara/out-of-core-reference.md index 740166dafc..655df0b0ee 100644 --- a/website/Map_Reduce_Folder/docs/latest/mahout-samsara/out-of-core-reference.md +++ b/website/Map_Reduce_Folder/docs/latest/mahout-samsara/out-of-core-reference.md @@ -14,7 +14,7 @@ The subjects of this reference are solely applicable to Mahout-Samsara's **DRM** In this reference, DRMs will be denoted as e.g. `A`, and in-core matrices as e.g. `inCoreA`. -#### Imports +#### Imports The following imports are used to enable seamless in-core and distributed algebraic DSL operations: @@ -23,16 +23,16 @@ The following imports are used to enable seamless in-core and distributed algebr import RLikeOps._ import drm._ import RLikeDRMOps._ - + If working with mixed scala/java code: - + import collection._ import JavaConversions._ - + If you are working with Mahout-Samsara's Spark-specific operations e.g. for context creation: import org.apache.mahout.sparkbindings._ - + The Mahout shell does all of these imports automatically. @@ -44,42 +44,42 @@ The Mahout shell does all of these imports automatically. Loading a DRM from (HD)FS: drmDfsRead(path = hdfsPath) - + Parallelizing from an in-core matrix: val inCoreA = (dense(1, 2, 3), (3, 4, 5)) val A = drmParallelize(inCoreA) - + Creating an empty DRM: val A = drmParallelizeEmpty(100, 50) - + Collecting to driver's jvm in-core: val inCoreA = A.collect - + **Warning: The collection of distributed matrices happens implicitly whenever conversion to an in-core (o.a.m.math.Matrix) type is required. E.g.:** val inCoreA: Matrix = ... val drmB: DrmLike[Int] =... val inCoreC: Matrix = inCoreA %*%: drmB - + **implies (incoreA %*%: drmB).collect** Collecting to (HD)FS as a Mahout's DRM formatted file: A.dfsWrite(path = hdfsPath) - + #### Logical algebraic operators on DRM matrices: A logical set of operators are defined for distributed matrices as a subset of those defined for in-core matrices. In particular, since all distributed matrices are immutable, there are no assignment operators (e.g. **A += B**) *Note: please see: [Mahout Scala Bindings and Mahout Spark Bindings for Linear Algebra Subroutines](http://mahout.apache.org/users/sparkbindings/ScalaSparkBindings.pdf) for information on Mahout-Samsars's Algebraic Optimizer, and translation from logical operations to a physical plan for the back end.* - - -Cache a DRM and trigger an optimized physical plan: + + +Cache a DRM and trigger an optimized physical plan: drmA.checkpoint(CacheHint.MEMORY_AND_DISK) - + Other valid caching Instructions: drmA.checkpoint(CacheHint.NONE) @@ -100,23 +100,23 @@ Other valid caching Instructions: Transposition: A.t - + Elementwise addition *(Matrices of identical geometry and row key types)*: - + A + B Elementwise subtraction *(Matrices of identical geometry and row key types)*: A - B - + Elementwise multiplication (Hadamard) *(Matrices of identical geometry and row key types)*: A * B - + Elementwise division *(Matrices of identical geometry and row key types)*: A / B - + **Elementwise operations involving one in-core argument (int-keyed DRMs only)**: A + inCoreB @@ -133,7 +133,7 @@ Elementwise division *(Matrices of identical geometry and row key types)*: inCoreA /: B Note the Spark associativity change (e.g. `A *: inCoreB` means `B.leftMultiply(A`), same as when both arguments are in core). Whenever operator arguments include both in-core and out-of-core arguments, the operator can only be associated with the out-of-core (DRM) argument to support the distributed implementation. - + **Matrix-matrix multiplication %*%**: `\(\mathbf{M}=\mathbf{AB}\)` @@ -145,13 +145,13 @@ Note the Spark associativity change (e.g. `A *: inCoreB` means `B.leftMultiply(A *Note: same as above, whenever operator arguments include both in-core and out-of-core arguments, the operator can only be associated with the out-of-core (DRM) argument to support the distributed implementation.* - + **Matrix-vector multiplication %*%** Currently we support a right multiply product of a DRM and an in-core Vector(`\(\mathbf{Ax}\)`) resulting in a single column DRM, which then can be collected in front (usually the desired outcome): val Ax = A %*% x val inCoreX = Ax.collect(::, 0) - + **Matrix-scalar +,-,*,/** Elementwise operations of every matrix element and a scalar: @@ -163,24 +163,24 @@ Elementwise operations of every matrix element and a scalar: A * 5.0 A / 5.0 5.0 /: a - + Note that `5.0 -: A` means `\(m_{ij} = 5 - a_{ij}\)` and `5.0 /: A` means `\(m_{ij} = \frac{5}{a{ij}}\)` for all elements of the result. - - + + #### Slicing General slice: A(100 to 200, 100 to 200) - + Horizontal Block: A(::, 100 to 200) - + Vertical Block: A(100 to 200, ::) - + *Note: if row range is not all-range (::) the the DRM must be `Int`-keyed. General case row slicing is not supported by DRMs with key types other than `Int`*. @@ -189,11 +189,11 @@ Vertical Block: Stitch side by side (cbind R semantics): val drmAnextToB = drmA cbind drmB - + Stitch side by side (Scala): val drmAnextToB = drmA.cbind(drmB) - + Analogously, vertical concatenation is available via **rbind** #### Custom pipelines on blocks @@ -210,7 +210,7 @@ Using `mapBlock` to add 1.0 to a DRM: val B = A.mapBlock() { case (keys, block) => keys -> (block += 1.0) } - + #### Broadcasting Vectors and matrices to closures Generally we can create and use one-way closure attributes to be used on the back end. @@ -229,7 +229,7 @@ Scalar matrix multiplication: val drm2 = drm1.mapBlock() { case (keys, block) => for(row <- 0 until block.nrow) block(row, ::) -= bcastV - keys -> block + keys -> block } #### Computations providing ad-hoc summaries @@ -241,32 +241,32 @@ Matrix cardinality: drmA.ncol *Note: depending on the stage of optimization, these may trigger a computational action. I.e. if one calls `nrow()` n times, then the back end will actually recompute `nrow` n times.* - + Means and sums: drmA.colSums drmA.colMeans drmA.rowSums drmA.rowMeans - - + + *Note: These will always trigger a computational action. I.e. if one calls `colSums()` n times, then the back end will actually recompute `colSums` n times.* #### Distributed Matrix Decompositions To import the decomposition package: - + import org.apache.mahout.math._ import decompositions._ - + Distributed thin QR: val (drmQ, incoreR) = dqrThin(drmA) - + Distributed SSVD: - + val (drmU, drmV, s) = dssvd(drmA, k = 40, q = 1) - + Distributed SPCA: val (drmU, drmV, s) = dspca(drmA, k = 30, q = 1) @@ -278,13 +278,13 @@ Distributed regularized ALS: lambda = 0.0, maxIterations = 10, convergenceThreshold = 0.10)) - + #### Adjusting parallelism of computations Set the minimum parallelism to 100 for computations on `drmA`: drmA.par(min = 100) - + Set the exact parallelism to 100 for computations on `drmA`: drmA.par(exact = 100) @@ -299,21 +299,14 @@ Set the engine specific automatic parallelism adjustment for computations on `dr **A Spark RDD:** val myRDD = drmA.checkpoint().rdd - + **An H2O Frame and Key Vec:** val myFrame = drmA.frame val myKeys = drmA.keys - + **A Flink DataSet:** val myDataSet = drmA.ds - -For more information including information on Mahout-Samsara's Algebraic Optimizer and in-core Linear algebra bindings see: [Mahout Scala Bindings and Mahout Spark Bindings for Linear Algebra Subroutines](http://mahout.apache.org/users/sparkbindings/ScalaSparkBindings.pdf) - - - - - - +For more information including information on Mahout-Samsara's Algebraic Optimizer and in-core Linear algebra bindings see: [Mahout Scala Bindings and Mahout Spark Bindings for Linear Algebra Subroutines](http://mahout.apache.org/users/sparkbindings/ScalaSparkBindings.pdf) diff --git a/website/Map_Reduce_Folder/docs/latest/quickstart.md b/website/Map_Reduce_Folder/docs/latest/quickstart.md index e963ce9076..242247d609 100644 --- a/website/Map_Reduce_Folder/docs/latest/quickstart.md +++ b/website/Map_Reduce_Folder/docs/latest/quickstart.md @@ -6,9 +6,9 @@ redirect_from: - /docs/latest/quickstart.html --- -# Mahout Quick Start +# Mahout Quick Start -The quickest way to take Apache Mahout for a test drive is by downloading a Docker container which contains Apache +The quickest way to take Apache Mahout for a test drive is by downloading a Docker container which contains Apache Zeppelin and is pre-loaded with a Spark-based, Mahout interpreter. See the instructions [here](http://mahout.apache.org/docs/latest/tutorials/misc/getting-started-with-zeppelin/). @@ -17,7 +17,7 @@ See the instructions [here](http://mahout.apache.org/docs/latest/tutorials/misc/ For a full list of Mahout's features see our [Features by Engine](http://mahout.apache.org/users/basics/algorithms.html) page. - + ## Using Mahout Mahout has prepared a bunch of examples and tutorials for users to quickly learn how to use its machine learning algorithms. @@ -36,10 +36,10 @@ Check the [Synthetic data](/users/clustering/clustering-of-synthetic-control-dat If you are interested in how to train a **Naive Bayes** model, look at the [20 newsgroups](/users/classification/twenty-newsgroups.html) example. -If you plan to build a **Hidden Markov Model** for speech recognition, the example [here](/users/classification/hidden-markov-models.html) might be instructive. +If you plan to build a **Hidden Markov Model** for speech recognition, the example [here](/users/classification/hidden-markov-models.html) might be instructive. Or you could build a **Random Forest** model by following this [quick start page](/users/classification/partial-implementation.html). -#### Working with Text +#### Working with Text If you need to convert raw text into word vectors as input to clustering or classification algorithms, please refer to this page on [how to create vectors from text](/users/basics/creating-vectors-from-text.html). diff --git a/website/Map_Reduce_Folder/docs/latest/tutorials/misc/getting-started-with-zepplin/index.md b/website/Map_Reduce_Folder/docs/latest/tutorials/misc/getting-started-with-zepplin/index.md index 1828974561..1f92edac49 100644 --- a/website/Map_Reduce_Folder/docs/latest/tutorials/misc/getting-started-with-zepplin/index.md +++ b/website/Map_Reduce_Folder/docs/latest/tutorials/misc/getting-started-with-zepplin/index.md @@ -7,8 +7,8 @@ redirect_from: --- Mahout has lots of pieces, and can be overwhelming to get started. We've tried to make it easier for users by providing -a precompiled Docker container which runs Apache Zeppelin (a popular notebooking tool) with an Apache Spark -interpreter that is preconfigured for Mahout and has all the required libraries loaded. We hope this will help you get +a precompiled Docker container which runs Apache Zeppelin (a popular notebooking tool) with an Apache Spark +interpreter that is preconfigured for Mahout and has all the required libraries loaded. We hope this will help you get "up and running with Mahout" as quickly as possible. (If you are interested in adding Apache Mahout to an existing Zeppelin Instance, see [this tutorial](http://mahout.apache.org/docs/latest/tutorials/misc/mahout-in-zeppelin/)). @@ -40,4 +40,4 @@ How easy was that?! (If you have troubles, make sure to sign up to user@mahout.a Next, check out the notebooks- one gives a nice primer to Apache Mahout already loaded. Enjoy! -**NOTE**: If you want to create your own Mahout on Zeppelin interpreter- please see [this page](http://mahout.apache.org/docs/latest/tutorials/misc/mahout-in-zeppelin/) \ No newline at end of file +**NOTE**: If you want to create your own Mahout on Zeppelin interpreter- please see [this page](http://mahout.apache.org/docs/latest/tutorials/misc/mahout-in-zeppelin/) diff --git a/website/Map_Reduce_Folder/environment/classify-a-doc-from-the-shell.md b/website/Map_Reduce_Folder/environment/classify-a-doc-from-the-shell.md index 4b050bb8e3..55be71866b 100644 --- a/website/Map_Reduce_Folder/environment/classify-a-doc-from-the-shell.md +++ b/website/Map_Reduce_Folder/environment/classify-a-doc-from-the-shell.md @@ -1,6 +1,6 @@ --- layout: default -title: +title: redirect_from: - /documentation/users/environment/classify-a-doc-from-the-shell - /documentation/users/environment/classify-a-doc-from-the-shell.html @@ -8,13 +8,13 @@ redirect_from: #Building a text classifier in Mahout's Spark Shell -This tutorial will take you through the steps used to train a Multinomial Naive Bayes model and create a text classifier based on that model using the ```mahout spark-shell```. +This tutorial will take you through the steps used to train a Multinomial Naive Bayes model and create a text classifier based on that model using the ```mahout spark-shell```. ## Prerequisites This tutorial assumes that you have your Spark environment variables set for the ```mahout spark-shell``` see: [Playing with Mahout's Shell](http://mahout.apache.org/users/sparkbindings/play-with-shell.html). As well we assume that Mahout is running in cluster mode (i.e. with the ```MAHOUT_LOCAL``` environment variable **unset**) as we'll be reading and writing to HDFS. ## Downloading and Vectorizing the Wikipedia dataset -*As of Mahout v. 0.10.0, we are still reliant on the MapReduce versions of ```mahout seqwiki``` and ```mahout seq2sparse``` to extract and vectorize our text. A* [*Spark implementation of seq2sparse*](https://issues.apache.org/jira/browse/MAHOUT-1663) *is in the works for Mahout v. 0.11.* However, to download the Wikipedia dataset, extract the bodies of the documentation, label each document and vectorize the text into TF-IDF vectors, we can simpmly run the [wikipedia-classifier.sh](https://github.com/apache/mahout/blob/master/examples/bin/classify-wikipedia.sh) example. +*As of Mahout v. 0.10.0, we are still reliant on the MapReduce versions of ```mahout seqwiki``` and ```mahout seq2sparse``` to extract and vectorize our text. A* [*Spark implementation of seq2sparse*](https://issues.apache.org/jira/browse/MAHOUT-1663) *is in the works for Mahout v. 0.11.* However, to download the Wikipedia dataset, extract the bodies of the documentation, label each document and vectorize the text into TF-IDF vectors, we can simpmly run the [wikipedia-classifier.sh](https://github.com/apache/mahout/blob/master/examples/bin/classify-wikipedia.sh) example. Please select a number to choose the corresponding task to run 1. CBayes (may require increased heap space on yarn) @@ -26,7 +26,7 @@ Enter (2). This will download a large recent XML dump of the Wikipedia database, ## Getting Started -Launch the ```mahout spark-shell```. There is an example script: ```spark-document-classifier.mscala``` (.mscala denotes a Mahout-Scala script which can be run similarly to an R script). We will be walking through this script for this tutorial but if you wanted to simply run the script, you could just issue the command: +Launch the ```mahout spark-shell```. There is an example script: ```spark-document-classifier.mscala``` (.mscala denotes a Mahout-Scala script which can be run similarly to an R script). We will be walking through this script for this tutorial but if you wanted to simply run the script, you could just issue the command: mahout> :load /path/to/mahout/examples/bin/spark-document-classifier.mscala @@ -61,11 +61,11 @@ Hadoop imports needed to read our dictionary: val model = SparkNaiveBayes.train(aggregatedObservations, labelIndex, false) val resAnalyzer = SparkNaiveBayes.test(model, fullData, false) println(resAnalyzer) - + printing the ```ResultAnalyzer``` will display the confusion matrix. ## Read in the dictionary and document frequency count from HDFS - + val dictionary = sdc.sequenceFile(pathToData + "wikipediaVecs/dictionary.file-0", classOf[Text], classOf[IntWritable]) @@ -74,16 +74,16 @@ printing the ```ResultAnalyzer``` will display the confusion matrix. classOf[LongWritable]) // setup the dictionary and document frequency count as maps - val dictionaryRDD = dictionary.map { + val dictionaryRDD = dictionary.map { case (wKey, wVal) => wKey.asInstanceOf[Text] - .toString() -> wVal.get() + .toString() -> wVal.get() } - + val documentFrequencyCountRDD = documentFrequencyCount.map { case (wKey, wVal) => wKey.asInstanceOf[IntWritable] - .get() -> wVal.get() + .get() -> wVal.get() } - + val dictionaryMap = dictionaryRDD.collect.map(x => x._1.toString -> x._2.toInt).toMap val dfCountMap = documentFrequencyCountRDD.collect.map(x => x._1.toInt -> x._2.toLong).toMap @@ -98,7 +98,7 @@ For this simple example, our function ```vectorizeDocument(...)``` will tokenize .toLowerCase .split(" ") .groupBy(identity) - .mapValues(_.length) + .mapValues(_.length) val vec = new RandomAccessSparseVector(dictionaryMap.size) val totalDFSize = dfMap(-1) val docSize = wordCounts.size @@ -124,17 +124,17 @@ For this simple example, our function ```vectorizeDocument(...)``` will tokenize val labelMap = model.labelIndex val numLabels = model.numLabels val reverseLabelMap = labelMap.map(x => x._2 -> x._1) - + // instantiate the correct type of classifier val classifier = model.isComplementary match { case true => new ComplementaryNBClassifier(model) case _ => new StandardNBClassifier(model) } -## Define an argmax function +## Define an argmax function The label with the highest score wins the classification for a given document. - + def argmax(v: Vector): (Int, Double) = { var bestIdx: Int = Integer.MIN_VALUE var bestScore: Double = Integer.MIN_VALUE.asInstanceOf[Int].toDouble @@ -156,7 +156,7 @@ The label with the highest score wins the classification for a given document. } ## Two sample news articles: United States Football and United Kingdom Football - + // A random United States football article // http://www.reuters.com/article/2015/01/28/us-nfl-superbowl-security-idUSKBN0L12JR20150128 val UStextToClassify = new String("(Reuters) - Super Bowl security officials acknowledge" + @@ -168,9 +168,9 @@ The label with the highest score wins the classification for a given document. " will battle. Deadly shootings in Paris and arrest of suspects in Belgium, Greece and" + " Germany heightened fears of more attacks around the world and social media accounts" + " linked to Middle East militant groups have carried a number of threats to attack" + - " high-profile U.S. events. There is no specific credible threat, said Johnson, who" + + " high-profile U.S. events. There is no specific credible threat, said Johnson, who" + " has appointed a federal coordination team to work with local, state and federal" + - " agencies to ensure safety of fans, players and other workers associated with the" + + " agencies to ensure safety of fans, players and other workers associated with the" + " Super Bowl. I'm confident we will have a safe and secure and successful event." + " Sunday's game has been given a Special Event Assessment Rating (SEAR) 1 rating, the" + " same as in previous years, except for the year after the Sept. 11, 2001 attacks, when" + @@ -190,7 +190,7 @@ The label with the highest score wins the classification for a given document. " every confidence the public safety agencies that represented in the planning process" + " are going to have their best and brightest out there this weekend and we will have" + " a very safe Super Bowl.") - + // A random United Kingdom football article // http://www.reuters.com/article/2015/01/26/manchester-united-swissquote-idUSL6N0V52RZ20150126 val UKtextToClassify = new String("(Reuters) - Manchester United have signed a sponsorship" + @@ -220,22 +220,22 @@ The label with the highest score wins the classification for a given document. " significant growth even within categories. United have endured a tricky transition" + " following the retirement of manager Alex Ferguson in 2013, finishing seventh in the" + " Premier League last season and missing out on a place in the lucrative Champions League." + - " ($1 = 0.8910 Swiss francs) (Writing by Neil Maidment, additional reporting by Jemima" + + " ($1 = 0.8910 Swiss francs) (Writing by Neil Maidment, additional reporting by Jemima" + " Kelly; editing by Keith Weir)") ## Vectorize and classify our documents val usVec = vectorizeDocument(UStextToClassify, dictionaryMap, dfCountMap) val ukVec = vectorizeDocument(UKtextToClassify, dictionaryMap, dfCountMap) - + println("Classifying the news article about superbowl security (united states)") classifyDocument(usVec) - + println("Classifying the news article about Manchester United (united kingdom)") classifyDocument(ukVec) -## Tie everything together in a new method to classify text - +## Tie everything together in a new method to classify text + def classifyText(txt: String): String = { val v = vectorizeDocument(txt, dictionaryMap, dfCountMap) classifyDocument(v) @@ -245,13 +245,13 @@ The label with the highest score wins the classification for a given document. classifyText("Hello world from Queens") classifyText("Hello world from London") - + ## Model persistance You can save the model to HDFS: model.dfsWrite("/path/to/model") - + And retrieve it with: val model = NBModel.dfsRead("/path/to/model") diff --git a/website/Map_Reduce_Folder/environment/h2o-internals.md b/website/Map_Reduce_Folder/environment/h2o-internals.md index 86a1ec19d0..6e2b260a38 100644 --- a/website/Map_Reduce_Folder/environment/h2o-internals.md +++ b/website/Map_Reduce_Folder/environment/h2o-internals.md @@ -1,6 +1,6 @@ --- layout: default -title: +title: redirect_from: - /documentation/users/environment/h2o-internals - /documentation/users/environment/h2o-internals.html diff --git a/website/Map_Reduce_Folder/environment/how-to-build-an-app.md b/website/Map_Reduce_Folder/environment/how-to-build-an-app.md index fa1f674833..149766641c 100644 --- a/website/Map_Reduce_Folder/environment/how-to-build-an-app.md +++ b/website/Map_Reduce_Folder/environment/how-to-build-an-app.md @@ -1,6 +1,6 @@ --- layout: default -title: +title: redirect_from: - /documentation/users/environment/how-to-build-an-app - /documentation/users/environment/how-to-build-an-app.html @@ -30,15 +30,15 @@ Using Mahout as a library in an application will require a little Scala code. Sc object CooccurrenceDriver extends App { } - + This will look a little different than Java since ```App``` does delayed initialization, which causes the body to be executed when the App is launched, just as in Java you would create a main method. Before we can execute something on Spark we'll need to create a context. We could use raw Spark calls here but default values are setup for a Mahout context by using the Mahout helper function. - implicit val mc = mahoutSparkContext(masterUrl = "local", + implicit val mc = mahoutSparkContext(masterUrl = "local", appName = "CooccurrenceDriver") - + We need to read in three files containing different interaction types. The files will each be read into a Mahout IndexedDataset. This allows us to preserve application-specific user and item IDs throughout the calculations. For example, here is data/purchase.csv: @@ -51,12 +51,12 @@ For example, here is data/purchase.csv: u4,iphone u4,galaxy -Mahout has a helper function that reads the text delimited files SparkEngine.indexedDatasetDFSReadElements. The function reads single element tuples (user-id,item-id) in a distributed way to create the IndexedDataset. Distributed Row Matrices (DRM) and Vectors are important data types supplied by Mahout and IndexedDataset is like a very lightweight Dataframe in R, it wraps a DRM with HashBiMaps for row and column IDs. +Mahout has a helper function that reads the text delimited files SparkEngine.indexedDatasetDFSReadElements. The function reads single element tuples (user-id,item-id) in a distributed way to create the IndexedDataset. Distributed Row Matrices (DRM) and Vectors are important data types supplied by Mahout and IndexedDataset is like a very lightweight Dataframe in R, it wraps a DRM with HashBiMaps for row and column IDs. One important thing to note about this example is that we read in all datasets before we adjust the number of rows in them to match the total number of users in the data. This is so the math works out [(A'A, A'B, A'C)](http://mahout.apache.org/users/algorithms/intro-cooccurrence-spark.html) even if some users took one action but not another there must be the same number of rows in all matrices. /** - * Read files of element tuples and create IndexedDatasets one per action. These + * Read files of element tuples and create IndexedDatasets one per action. These * share a userID BiMap but have their own itemID BiMaps */ def readActions(actionInput: Array[(String, String)]): Array[(String, IndexedDataset)] = { @@ -64,7 +64,7 @@ One important thing to note about this example is that we read in all datasets b val userDictionary: BiMap[String, Int] = HashBiMap.create() - // The first action named in the sequence is the "primary" action and + // The first action named in the sequence is the "primary" action and // begins to fill up the user dictionary for ( actionDescription <- actionInput ) {// grab the path to actions val action: IndexedDataset = SparkEngine.indexedDatasetDFSReadElements( @@ -73,11 +73,11 @@ One important thing to note about this example is that we read in all datasets b existingRowIDs = userDictionary) userDictionary.putAll(action.rowIDs) // put the name in the tuple with the indexedDataset - actions = actions :+ (actionDescription._1, action) + actions = actions :+ (actionDescription._1, action) } - // After all actions are read in the userDictonary will contain every user seen, - // even if they may not have taken all actions . Now we adjust the row rank of + // After all actions are read in the userDictonary will contain every user seen, + // even if they may not have taken all actions . Now we adjust the row rank of // all IndxedDataset's to have this number of rows // Note: this is very important or the cooccurrence calc may fail val numUsers = userDictionary.size() // one more than the cardinality @@ -95,7 +95,7 @@ Now that we have the data read in we can perform the cooccurrence calculation. // actions.map creates an array of just the IndeedDatasets val indicatorMatrices = SimilarityAnalysis.cooccurrencesIDSs( - actions.map(a => a._2)) + actions.map(a => a._2)) All we need to do now is write the indicators. @@ -116,14 +116,14 @@ The ```writeIndicators``` method uses the default write function ```dfsWrite```. val indicatorDir = OutputPath + indicator._1 indicator._2.dfsWrite( indicatorDir, - // Schema tells the writer to omit LLR strengths + // Schema tells the writer to omit LLR strengths // and format for search engine indexing - IndexedDatasetWriteBooleanSchema) + IndexedDatasetWriteBooleanSchema) } } - -See the Github project for the full source. Now we create a build.sbt to build the example. + +See the Github project for the full source. Now we create a build.sbt to build the example. name := "cooccurrence-driver" @@ -164,7 +164,7 @@ Building the examples from project's root folder: This will automatically set up some launcher scripts for the driver. To run execute $ target/pack/bin/cooc - + The driver will execute in Spark standalone mode and put the data in /path/to/3-input-cooc/data/indicators/*indicator-type* ##Using a Debugger @@ -172,7 +172,7 @@ To build and run this example in a debugger like IntelliJ IDEA. Install from the Open IDEA and go to the menu File->New->Project from existing sources->SBT->/path/to/3-input-cooc. This will create an IDEA project from ```build.sbt``` in the root directory. -At this point you may create a "Debug Configuration" to run. In the menu choose Run->Edit Configurations. Under "Default" choose "Application". In the dialog hit the elipsis button "..." to the right of "Environment Variables" and fill in your versions of JAVA_HOME, SPARK_HOME, and MAHOUT_HOME. In configuration editor under "Use classpath from" choose root-3-input-cooc module. +At this point you may create a "Debug Configuration" to run. In the menu choose Run->Edit Configurations. Under "Default" choose "Application". In the dialog hit the elipsis button "..." to the right of "Environment Variables" and fill in your versions of JAVA_HOME, SPARK_HOME, and MAHOUT_HOME. In configuration editor under "Use classpath from" choose root-3-input-cooc module. ![image](http://mahout.apache.org/images/debug-config.png) @@ -191,18 +191,18 @@ To make the CooccurrenceDriver.scala into a script make the following changes: * You won't need the context, since it is created when the shell is launched, comment that line out. * Replace the logger.info lines with println -* Remove the package info since it's not needed, this will produce the file in ```path/to/3-input-cooc/bin/CooccurrenceDriver.mscala```. +* Remove the package info since it's not needed, this will produce the file in ```path/to/3-input-cooc/bin/CooccurrenceDriver.mscala```. Note the extension ```.mscala``` to indicate we are using Mahout's scala extensions for math, otherwise known as [Mahout-Samsara](http://mahout.apache.org/users/environment/out-of-core-reference.html) To run the code make sure the output does not exist already $ rm -r /path/to/3-input-cooc/data/indicators - + Launch the Mahout + Spark shell: $ mahout spark-shell - + You'll see the Mahout splash: MAHOUT_LOCAL is set, so we don't add HADOOP_CONF_DIR to classpath. @@ -213,14 +213,14 @@ You'll see the Mahout splash: | | | | | | (_| | | | | (_) | |_| | |_ |_| |_| |_|\__,_|_| |_|\___/ \__,_|\__| version 0.10.0 - + Using Scala version 2.10.4 (Java HotSpot(TM) 64-Bit Server VM, Java 1.7.0_72) Type in expressions to have them evaluated. Type :help for more information. 15/04/26 09:30:48 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable Created spark context.. Mahout distributed context is available as "implicit val sdc". - mahout> + mahout> To load the driver type: @@ -233,12 +233,12 @@ To load the driver type: import org.apache.mahout.sparkbindings._ import scala.collection.immutable.HashMap defined module CooccurrenceDriver - mahout> + mahout> To run the driver type: mahout> CooccurrenceDriver.main(args = Array("")) - + You'll get some stats printed: Total number of users for all actions = 5 @@ -254,5 +254,5 @@ You'll get some stats printed: Number of rows for matrix = 5 Number of columns for matrix = 7 Number of rows after resize = 5 - + If you look in ```path/to/3-input-cooc/data/indicators``` you should find folders containing the indicator matrices. diff --git a/website/Map_Reduce_Folder/environment/in-core-reference.md b/website/Map_Reduce_Folder/environment/in-core-reference.md index 4858bdc896..b2f9ad9ae5 100644 --- a/website/Map_Reduce_Folder/environment/in-core-reference.md +++ b/website/Map_Reduce_Folder/environment/in-core-reference.md @@ -1,6 +1,6 @@ --- layout: default -title: +title: redirect_from: - /documentation/users/environment/in-core-reference - /documentation/users/environment/in-core-reference.html @@ -15,7 +15,7 @@ The following imports are used to enable Mahout-Samsara's Scala DSL bindings for import org.apache.mahout.math._ import scalabindings._ import RLikeOps._ - + #### Inline initalization Dense vectors: @@ -30,13 +30,13 @@ Sparse vectors: // to create a vector with specific cardinality val sparseVec1 = svec((5 -> 1.0) :: (10 -> 2.0) :: Nil, cardinality = 20) - -Inline matrix initialization, either sparse or dense, is always done row wise. + +Inline matrix initialization, either sparse or dense, is always done row wise. Dense matrices: val A = dense((1, 2, 3), (3, 4, 5)) - + Sparse matrices: val A = sparse( @@ -51,11 +51,11 @@ Diagonal matrix with constant diagonal elements: Diagonal matrix with main diagonal backed by a vector: diagv((1, 2, 3, 4, 5)) - + Identity matrix: eye(10) - + ####Slicing and Assigning Getting a vector element: @@ -63,60 +63,60 @@ Getting a vector element: val d = vec(5) Setting a vector element: - + vec(5) = 3.0 - + Getting a matrix element: val d = m(3,5) - + Setting a matrix element: M(3,5) = 3.0 - + Getting a matrix row or column: val rowVec = M(3, ::) val colVec = M(::, 3) - + Setting a matrix row or column via vector assignment: M(3, ::) := (1, 2, 3) M(::, 3) := (1, 2, 3) - + Setting a subslices of a matrix row or column: a(0, 0 to 1) = (3, 5) - + Setting a subslices of a matrix row or column via vector assignment: a(0, 0 to 1) := (3, 5) - + Getting a matrix as from matrix contiguous block: val B = A(2 to 3, 3 to 4) - + Assigning a contiguous block to a matrix: A(0 to 1, 1 to 2) = dense((3, 2), (3 ,3)) - + Assigning a contiguous block to a matrix using the matrix assignment operator: A(o to 1, 1 to 2) := dense((3, 2), (3, 3)) - + Assignment operator used for copying between vectors or matrices: vec1 := vec2 M1 := M2 - + Assignment operator using assignment through a functional literal for a matrix: M := ((row, col, x) => if (row == col) 1 else 0 - + Assignment operator using assignment through a functional literal for a vector: vec := ((index, x) => sqrt(x) - + #### BLAS-like operations Plus/minus either vector or numeric with assignment or not: @@ -125,7 +125,7 @@ Plus/minus either vector or numeric with assignment or not: a - b a + 5.0 a - 5.0 - + Hadamard (elementwise) product, either vector or matrix or numeric operands: a * b @@ -139,41 +139,41 @@ Operations with assignment: a -= 5.0 a *= b a *= 5 - -*Some nuanced rules*: + +*Some nuanced rules*: 1/x in R (where x is a vector or a matrix) is elementwise inverse. In scala it would be expressed as: val xInv = 1 /: x and R's 5.0 - x would be: - + val x1 = 5.0 -: x - + *note: All assignment operations, including :=, return the assignee just like in C++*: - a -= b - -assigns **a - b** to **b** (in-place) and returns **b**. Similarly for **a /=: b** or **1 /=: v** - + a -= b + +assigns **a - b** to **b** (in-place) and returns **b**. Similarly for **a /=: b** or **1 /=: v** + Dot product: a dot b - + Matrix and vector equivalency (or non-equivalency). **Dangerous, exact equivalence is rarely useful, better to use norm comparisons with an allowance of small errors.** - + a === b a !== b - -Matrix multiply: + +Matrix multiply: a %*% b - -Optimized Right Multiply with a diagonal matrix: + +Optimized Right Multiply with a diagonal matrix: diag(5, 5) :%*% b - + Optimized Left Multiply with a diagonal matrix: A %*%: diag(5, 5) @@ -181,15 +181,15 @@ Optimized Left Multiply with a diagonal matrix: Second norm, of a vector or matrix: a.norm - + Transpose: val Mt = M.t - + *note: Transposition is currently handled via view, i.e. updating a transposed matrix will be updating the original.* Also computing something like `\(\mathbf{X^\top}\mathbf{X}\)`: val XtX = X.t %*% X - + will not therefore incur any additional data copying. #### Decompositions @@ -204,71 +204,71 @@ All arguments in the following are matricies. **Cholesky decomposition** val ch = chol(M) - + **SVD** val (U, V, s) = svd(M) - + **EigenDecomposition** val (V, d) = eigen(M) - + **QR decomposition** val (Q, R) = qr(M) - + **Rank**: Check for rank deficiency (runs rank-revealing QR) M.isFullRank - + **In-core SSVD** Val (U, V, s) = ssvd(A, k = 50, p = 15, q = 1) - + **Solving linear equation systems and matrix inversion:** fully similar to R semantics; there are three forms of invocation: Solve `\(\mathbf{AX}=\mathbf{B}\)`: solve(A, B) - + Solve `\(\mathbf{Ax}=\mathbf{b}\)`: - + solve(A, b) - + Compute `\(\mathbf{A^{-1}}\)`: solve(A) - + #### Misc Vector cardinality: a.length - + Matrix cardinality: m.nrow m.ncol - + Means and sums: m.colSums m.colMeans m.rowSums m.rowMeans - + Copy-By-Value: val b = a cloned - + #### Random Matrices `\(\mathcal{U}\)`(0,1) random matrix view: val incCoreA = Matrices.uniformView(m, n, seed) - + `\(\mathcal{U}\)`(-1,1) random matrix view: val incCoreA = Matrices.symmetricUniformView(m, n, seed) @@ -276,30 +276,28 @@ Copy-By-Value: `\(\mathcal{N}\)`(-1,1) random matrix view: val incCoreA = Matrices.gaussianView(m, n, seed) - -#### Iterators + +#### Iterators Mahout-Math already exposes a number of iterators. Scala code just needs the following imports to enable implicit conversions to scala iterators. import collection._ import JavaConversions._ - + Iterating over rows in a Matrix: for (row <- m) { ... do something with row } - + For more information including information on Mahout-Samsara's out-of-core Linear algebra bindings see: [Mahout Scala Bindings and Mahout Spark Bindings for Linear Algebra Subroutines](http://mahout.apache.org/users/sparkbindings/ScalaSparkBindings.pdf) - - diff --git a/website/Map_Reduce_Folder/environment/out-of-core-reference.md b/website/Map_Reduce_Folder/environment/out-of-core-reference.md index 51d0edc1d5..6ef4d48204 100644 --- a/website/Map_Reduce_Folder/environment/out-of-core-reference.md +++ b/website/Map_Reduce_Folder/environment/out-of-core-reference.md @@ -1,6 +1,6 @@ --- layout: default -title: +title: redirect_from: - /documentation/users/environment/out-of-core-reference - /documentation/users/environment/out-of-core-reference.html @@ -14,7 +14,7 @@ The subjects of this reference are solely applicable to Mahout-Samsara's **DRM** In this reference, DRMs will be denoted as e.g. `A`, and in-core matrices as e.g. `inCoreA`. -#### Imports +#### Imports The following imports are used to enable seamless in-core and distributed algebraic DSL operations: @@ -23,16 +23,16 @@ The following imports are used to enable seamless in-core and distributed algebr import RLikeOps._ import drm._ import RLikeDRMOps._ - + If working with mixed scala/java code: - + import collection._ import JavaConversions._ - + If you are working with Mahout-Samsara's Spark-specific operations e.g. for context creation: import org.apache.mahout.sparkbindings._ - + The Mahout shell does all of these imports automatically. @@ -44,42 +44,42 @@ The Mahout shell does all of these imports automatically. Loading a DRM from (HD)FS: drmDfsRead(path = hdfsPath) - + Parallelizing from an in-core matrix: val inCoreA = (dense(1, 2, 3), (3, 4, 5)) val A = drmParallelize(inCoreA) - + Creating an empty DRM: val A = drmParallelizeEmpty(100, 50) - + Collecting to driver's jvm in-core: val inCoreA = A.collect - + **Warning: The collection of distributed matrices happens implicitly whenever conversion to an in-core (o.a.m.math.Matrix) type is required. E.g.:** val inCoreA: Matrix = ... val drmB: DrmLike[Int] =... val inCoreC: Matrix = inCoreA %*%: drmB - + **implies (incoreA %*%: drmB).collect** Collecting to (HD)FS as a Mahout's DRM formatted file: A.dfsWrite(path = hdfsPath) - + #### Logical algebraic operators on DRM matrices: A logical set of operators are defined for distributed matrices as a subset of those defined for in-core matrices. In particular, since all distributed matrices are immutable, there are no assignment operators (e.g. **A += B**) *Note: please see: [Mahout Scala Bindings and Mahout Spark Bindings for Linear Algebra Subroutines](http://mahout.apache.org/users/sparkbindings/ScalaSparkBindings.pdf) for information on Mahout-Samsars's Algebraic Optimizer, and translation from logical operations to a physical plan for the back end.* - - -Cache a DRM and trigger an optimized physical plan: + + +Cache a DRM and trigger an optimized physical plan: drmA.checkpoint(CacheHint.MEMORY_AND_DISK) - + Other valid caching Instructions: drmA.checkpoint(CacheHint.NONE) @@ -100,23 +100,23 @@ Other valid caching Instructions: Transposition: A.t - + Elementwise addition *(Matrices of identical geometry and row key types)*: - + A + B Elementwise subtraction *(Matrices of identical geometry and row key types)*: A - B - + Elementwise multiplication (Hadamard) *(Matrices of identical geometry and row key types)*: A * B - + Elementwise division *(Matrices of identical geometry and row key types)*: A / B - + **Elementwise operations involving one in-core argument (int-keyed DRMs only)**: A + inCoreB @@ -133,7 +133,7 @@ Elementwise division *(Matrices of identical geometry and row key types)*: inCoreA /: B Note the Spark associativity change (e.g. `A *: inCoreB` means `B.leftMultiply(A`), same as when both arguments are in core). Whenever operator arguments include both in-core and out-of-core arguments, the operator can only be associated with the out-of-core (DRM) argument to support the distributed implementation. - + **Matrix-matrix multiplication %*%**: `\(\mathbf{M}=\mathbf{AB}\)` @@ -145,13 +145,13 @@ Note the Spark associativity change (e.g. `A *: inCoreB` means `B.leftMultiply(A *Note: same as above, whenever operator arguments include both in-core and out-of-core arguments, the operator can only be associated with the out-of-core (DRM) argument to support the distributed implementation.* - + **Matrix-vector multiplication %*%** Currently we support a right multiply product of a DRM and an in-core Vector(`\(\mathbf{Ax}\)`) resulting in a single column DRM, which then can be collected in front (usually the desired outcome): val Ax = A %*% x val inCoreX = Ax.collect(::, 0) - + **Matrix-scalar +,-,*,/** Elementwise operations of every matrix element and a scalar: @@ -163,24 +163,24 @@ Elementwise operations of every matrix element and a scalar: A * 5.0 A / 5.0 5.0 /: a - + Note that `5.0 -: A` means `\(m_{ij} = 5 - a_{ij}\)` and `5.0 /: A` means `\(m_{ij} = \frac{5}{a{ij}}\)` for all elements of the result. - - + + #### Slicing General slice: A(100 to 200, 100 to 200) - + Horizontal Block: A(::, 100 to 200) - + Vertical Block: A(100 to 200, ::) - + *Note: if row range is not all-range (::) the the DRM must be `Int`-keyed. General case row slicing is not supported by DRMs with key types other than `Int`*. @@ -189,11 +189,11 @@ Vertical Block: Stitch side by side (cbind R semantics): val drmAnextToB = drmA cbind drmB - + Stitch side by side (Scala): val drmAnextToB = drmA.cbind(drmB) - + Analogously, vertical concatenation is available via **rbind** #### Custom pipelines on blocks @@ -210,7 +210,7 @@ Using `mapBlock` to add 1.0 to a DRM: val B = A.mapBlock() { case (keys, block) => keys -> (block += 1.0) } - + #### Broadcasting Vectors and matrices to closures Generally we can create and use one-way closure attributes to be used on the back end. @@ -229,7 +229,7 @@ Scalar matrix multiplication: val drm2 = drm1.mapBlock() { case (keys, block) => for(row <- 0 until block.nrow) block(row, ::) -= bcastV - keys -> block + keys -> block } #### Computations providing ad-hoc summaries @@ -241,32 +241,32 @@ Matrix cardinality: drmA.ncol *Note: depending on the stage of optimization, these may trigger a computational action. I.e. if one calls `nrow()` n times, then the back end will actually recompute `nrow` n times.* - + Means and sums: drmA.colSums drmA.colMeans drmA.rowSums drmA.rowMeans - - + + *Note: These will always trigger a computational action. I.e. if one calls `colSums()` n times, then the back end will actually recompute `colSums` n times.* #### Distributed Matrix Decompositions To import the decomposition package: - + import org.apache.mahout.math._ import decompositions._ - + Distributed thin QR: val (drmQ, incoreR) = dqrThin(drmA) - + Distributed SSVD: - + val (drmU, drmV, s) = dssvd(drmA, k = 40, q = 1) - + Distributed SPCA: val (drmU, drmV, s) = dspca(drmA, k = 30, q = 1) @@ -278,13 +278,13 @@ Distributed regularized ALS: lambda = 0.0, maxIterations = 10, convergenceThreshold = 0.10)) - + #### Adjusting parallelism of computations Set the minimum parallelism to 100 for computations on `drmA`: drmA.par(min = 100) - + Set the exact parallelism to 100 for computations on `drmA`: drmA.par(exact = 100) @@ -299,21 +299,14 @@ Set the engine specific automatic parallelism adjustment for computations on `dr **A Spark RDD:** val myRDD = drmA.checkpoint().rdd - + **An H2O Frame and Key Vec:** val myFrame = drmA.frame val myKeys = drmA.keys - + **A Flink DataSet:** val myDataSet = drmA.ds - -For more information including information on Mahout-Samsara's Algebraic Optimizer and in-core Linear algebra bindings see: [Mahout Scala Bindings and Mahout Spark Bindings for Linear Algebra Subroutines](http://mahout.apache.org/users/sparkbindings/ScalaSparkBindings.pdf) - - - - - - +For more information including information on Mahout-Samsara's Algebraic Optimizer and in-core Linear algebra bindings see: [Mahout Scala Bindings and Mahout Spark Bindings for Linear Algebra Subroutines](http://mahout.apache.org/users/sparkbindings/ScalaSparkBindings.pdf) diff --git a/website/Map_Reduce_Folder/environment/spark-internals.md b/website/Map_Reduce_Folder/environment/spark-internals.md index cac58941d6..074bfb7cd9 100644 --- a/website/Map_Reduce_Folder/environment/spark-internals.md +++ b/website/Map_Reduce_Folder/environment/spark-internals.md @@ -1,6 +1,6 @@ --- layout: default -title: +title: redirect_from: - /documentation/users/environment/spark-internals - /documentation/users/environment/spark-internals.html @@ -8,7 +8,7 @@ redirect_from: # Introduction -This document provides an overview of how the Mahout Scala DSL (distributed algebraic operators) is implemented over the Spark back end engine. The document is aimed at Mahout developers, to give a high level description of the design. +This document provides an overview of how the Mahout Scala DSL (distributed algebraic operators) is implemented over the Spark back end engine. The document is aimed at Mahout developers, to give a high level description of the design. ## Spark Overview diff --git a/website/Map_Reduce_Folder/flinkbindings/flink-internals.md b/website/Map_Reduce_Folder/flinkbindings/flink-internals.md index 6504cfc32a..f20710f0e9 100644 --- a/website/Map_Reduce_Folder/flinkbindings/flink-internals.md +++ b/website/Map_Reduce_Folder/flinkbindings/flink-internals.md @@ -1,6 +1,6 @@ --- layout: default -title: +title: redirect_from: - /documentation/users/flinkbindings/flink-internals - /documentation/users/flinkbindings/flink-internals.html @@ -18,7 +18,7 @@ The Mahout DRM, or Distributed Row Matrix, is an abstraction for storing a large #Flink Overview -Apache Flink is an open source, distributed Stream and Batch Processing Framework. At it's core, Flink is a Stream Processing engine and Batch processing is an extension of Stream Processing. +Apache Flink is an open source, distributed Stream and Batch Processing Framework. At it's core, Flink is a Stream Processing engine and Batch processing is an extension of Stream Processing. Flink includes several APIs for building applications with the Flink Engine: @@ -47,5 +47,3 @@ flink/src/main/scala/org/apache/mahout/flinkbindings/drm/*.scala - Flink Dataset flink/src/main/scala/org/apache/mahout/flinkbindings/io/*.scala - Read / Write between DRMDataSet and files on HDFS flink/src/main/scala/org/apache/mahout/flinkbindings/FlinkEngine.scala - DSL operator graph evaluator and various abstract API implementations for a distributed engine. - - diff --git a/website/Map_Reduce_Folder/flinkbindings/playing-with-samsara-flink.md b/website/Map_Reduce_Folder/flinkbindings/playing-with-samsara-flink.md index 614bc8fdd1..1343b7e164 100644 --- a/website/Map_Reduce_Folder/flinkbindings/playing-with-samsara-flink.md +++ b/website/Map_Reduce_Folder/flinkbindings/playing-with-samsara-flink.md @@ -1,12 +1,12 @@ --- layout: default -title: +title: redirect_from: - /documentation/users/flinkbindings/playing-with-samsara-flink - /documentation/users/flinkbindings/playing-with-samsara-flink.html --- -## Getting Started +## Getting Started To get started, add the following dependency to the pom: @@ -45,18 +45,18 @@ The top JIRA for Flink backend is [MAHOUT-1570](https://issues.apache.org/jira/b ### Implemented * [MAHOUT-1701](https://issues.apache.org/jira/browse/MAHOUT-1701) Mahout DSL for Flink: implement AtB ABt and AtA operators -* [MAHOUT-1702](https://issues.apache.org/jira/browse/MAHOUT-1702) implement element-wise operators (like `A + 2` or `A + B`) +* [MAHOUT-1702](https://issues.apache.org/jira/browse/MAHOUT-1702) implement element-wise operators (like `A + 2` or `A + B`) * [MAHOUT-1703](https://issues.apache.org/jira/browse/MAHOUT-1703) implement `cbind` and `rbind` * [MAHOUT-1709](https://issues.apache.org/jira/browse/MAHOUT-1709) implement slicing (like `A(1 to 10, ::)`) -* [MAHOUT-1710](https://issues.apache.org/jira/browse/MAHOUT-1710) implement right in-core matrix multiplication (`A %*% B` when `B` is in-core) +* [MAHOUT-1710](https://issues.apache.org/jira/browse/MAHOUT-1710) implement right in-core matrix multiplication (`A %*% B` when `B` is in-core) * [MAHOUT-1711](https://issues.apache.org/jira/browse/MAHOUT-1711) implement broadcasting * [MAHOUT-1712](https://issues.apache.org/jira/browse/MAHOUT-1712) implement operators `At`, `Ax`, `Atx` - `Ax` and `At` are implemented * [MAHOUT-1734](https://issues.apache.org/jira/browse/MAHOUT-1734) implement I/O - should be able to read results of Flink bindings * [MAHOUT-1747](https://issues.apache.org/jira/browse/MAHOUT-1747) add support for different types of indexes (String, long, etc) - now supports `Int`, `Long` and `String` -* [MAHOUT-1748](https://issues.apache.org/jira/browse/MAHOUT-1748) switch to Flink Scala API +* [MAHOUT-1748](https://issues.apache.org/jira/browse/MAHOUT-1748) switch to Flink Scala API * [MAHOUT-1749](https://issues.apache.org/jira/browse/MAHOUT-1749) Implement `Atx` * [MAHOUT-1750](https://issues.apache.org/jira/browse/MAHOUT-1750) Implement `ABt` -* [MAHOUT-1751](https://issues.apache.org/jira/browse/MAHOUT-1751) Implement `AtA` +* [MAHOUT-1751](https://issues.apache.org/jira/browse/MAHOUT-1751) Implement `AtA` * [MAHOUT-1755](https://issues.apache.org/jira/browse/MAHOUT-1755) Flush intermediate results to FS - Flink, unlike Spark, does not store intermediate results in memory. * [MAHOUT-1764](https://issues.apache.org/jira/browse/MAHOUT-1764) Add standard backend tests for Flink * [MAHOUT-1765](https://issues.apache.org/jira/browse/MAHOUT-1765) Add documentation about Flink backend @@ -80,14 +80,14 @@ The top JIRA for Flink backend is [MAHOUT-1570](https://issues.apache.org/jira/b * [MAHOUT-1824](https://issues.apache.org/jira/browse/MAHOUT-1824) Optimize FlinkOpAtA to use upper triangular matrices * [MAHOUT-1825](https://issues.apache.org/jira/browse/MAHOUT-1825) Add List of Flink algorithms to Mahout wiki page -### Tests +### Tests -There is a set of standard tests that all engines should pass (see [MAHOUT-1764](https://issues.apache.org/jira/browse/MAHOUT-1764)). +There is a set of standard tests that all engines should pass (see [MAHOUT-1764](https://issues.apache.org/jira/browse/MAHOUT-1764)). -* `DistributedDecompositionsSuite` -* `DrmLikeOpsSuite` -* `DrmLikeSuite` -* `RLikeDrmOpsSuite` +* `DistributedDecompositionsSuite` +* `DrmLikeOpsSuite` +* `DrmLikeSuite` +* `RLikeDrmOpsSuite` These are Flink-backend specific tests, e.g. @@ -97,14 +97,14 @@ These are Flink-backend specific tests, e.g. * `LATestSuite` tests for specific operators like `AtB`, `Ax`, etc * `UseCasesSuite` has more complex examples, like power iteration, ridge regression, etc -## Environment +## Environment -For development the minimal supported configuration is +For development the minimal supported configuration is * [JDK 1.7](http://www.oracle.com/technetwork/java/javase/downloads/jdk7-downloads-1880260.html) * [Scala 2.10] -When using mahout, please import the following modules: +When using mahout, please import the following modules: * `mahout-math` * `mahout-math-scala` diff --git a/website/Map_Reduce_Folder/general/books-tutorials-and-talks.md b/website/Map_Reduce_Folder/general/books-tutorials-and-talks.md index a62dcd0c59..3e4993361f 100644 --- a/website/Map_Reduce_Folder/general/books-tutorials-and-talks.md +++ b/website/Map_Reduce_Folder/general/books-tutorials-and-talks.md @@ -34,7 +34,7 @@ given. # News, Articles and Tutorials - * [Mahout 0.10.x: first Mahout release as a programming environment](http://www.weatheringthroughtechdays.com/2015/04/mahout-010x-first-mahout-release-as.html) + * [Mahout 0.10.x: first Mahout release as a programming environment](http://www.weatheringthroughtechdays.com/2015/04/mahout-010x-first-mahout-release-as.html) * [Comparing Document Classification Functions of Lucene and Mahout](http://soleami.com/blog/comparing-document-classification-functions-of-lucene-and-mahout.html) * Apache Mahout: Scalable Machine Learning for Everyone * How to build a spam filter server with Mahout - Applying classification on a live server - April 2011 @@ -65,11 +65,11 @@ In reverse chronological order, so that most recent talks are at the top * [Distributed Machine Learning with Apache Mahout] Suneel Marthi at Apache Big Data North America, Vancouver, Canada, May 11, 2016 and MapR Washington DC Big Data Everywhere, Tysons, VA, June 2 2016 * [Declarative Machine Learning with the Samsara DSL](http://www.slideshare.net/FlinkForward/sebastian-schelter-distributed-machine-learing-with-the-samsara-dsl) Sebastian Schelter at Flink Forward Conference, Berlin Germany, October 2015. * [Bringing Algebraic Semantics to Mahout](http://www.slideshare.net/sscdotopen/bringing-algebraic-semantics-to-mahout) Sebastian Schelter at HPI Infolunch, Potsdam Germany, May 2014 - * Mahout Spark and Scala bindings: Bringing Algebraic Semantics ([slides](http://www.slideshare.net/DmitriyLyubimov/mahout-scala-and-spark-bindings)/[video](http://youtu.be/h9dpmvNW1Dw)) - Dmitriy Lyubimov at Mahout Meetup, April 17, 2014. + * Mahout Spark and Scala bindings: Bringing Algebraic Semantics ([slides](http://www.slideshare.net/DmitriyLyubimov/mahout-scala-and-spark-bindings)/[video](http://youtu.be/h9dpmvNW1Dw)) - Dmitriy Lyubimov at Mahout Meetup, April 17, 2014. * Mahout Future Directions - Ted Dunning, Suneel Marthi, Sebastian Schelter at Hadoop Summit Europe 2014, Amsterdam, April 3, 2014 * Building Recommender Systems for Mere-Mortals - Sebastian Schelter at Researchgate Developer Day, Berlin, November 2013 * Recommendations with Apache Mahout - Sebastian Schelter at IBM Almaden Research Center, San Jose, September 2013 - * Next Directions in Mahout’s Recommenders - Sebastian Schelter at Bay Area Mahout Meetup, Redwood City, August 2013 + * Next Directions in Mahout’s Recommenders - Sebastian Schelter at Bay Area Mahout Meetup, Redwood City, August 2013 * New Directions in Mahout’s Recommenders - Sebastian Schelter at Recommender Systems Get Together Berlin, April 2013 * Introduction to Mahout and Machine Learning - Slides by Varad Meru, Software Development Engineer at Orzota. July 27th, 2013. * An Introduction to Collaborative Filtering with Apache Mahout - Sebastian Schelter at Recommender Systems Challenge Workshop in conjunction with ACM RecSys 2012, Dublin, September 2012 @@ -82,7 +82,7 @@ In reverse chronological order, so that most recent talks are at the top * First Mahout Hackathon, Berlin, March 2011 * Mahout meetup - there were two talks at the Apache Mahout meetup at JTeam in Amsterdam, February 2011. intro slides * Mahout clustering - Talk on Mahout clustering at data dev room FOSDEM, February 2011. - * Scaling Data Analysis with Apache Mahout - talk on Mahout at O'Reilly Strata, February 2011. + * Scaling Data Analysis with Apache Mahout - talk on Mahout at O'Reilly Strata, February 2011. * Practical Machine Learning - Slides from Biju B and Jaganadh G, FOSSMEET-NITC, Calicut, India, February 2011. * Mahout at AlphaCSPs The Edge 2010 (pdf) - slideshare - Slides from Ariel Kogan AlphaCSP's The Edge, December 2010. * Intelligent data analysis with Apache Mahout - Slides from Isabel Drost, Devoxx Antwerp, November 2010. @@ -113,7 +113,7 @@ In reverse chronological order, so that most recent talks are at the top * BI Over Petabytes: Meet Apache Mahout - Slides from a talk by Jeff Eastman on April 21, 2009 at the Bay Area SD Forum Business Intelligence SIG meeting at SAP in Palo Alto, CA. * Lucene Meetup and Apache Barcamp in Amsterdam, March 2009. * BarCampRDU - (Raleigh) on Aug. 2, 2008 - * Introducing Mahout: Apache Machine Learning - Committer Grant Ingersoll gave a gentle introduction to Mahout and Machine Learning at ApacheCon in November (3rd through 7th) in New Orleans, USA. + * Introducing Mahout: Apache Machine Learning - Committer Grant Ingersoll gave a gentle introduction to Mahout and Machine Learning at ApacheCon in November (3rd through 7th) in New Orleans, USA. * Mahout: Scaling Machine Learning - Introduction to Mahout and machine learning at FrOSCon in Sankt Augustin/Germany, Isabel Drost, August 2008. (slides) * Mahout: Scalable Machine Learning - An introduction to Mahout and machine learning at the first German Hadoop gathering in newthinking store/ Berlin, Isabel Drost, July 2008. * Apache Mahout: Industrial Strength Machine Learning - Committer Jeff Eastman gave an introduction to Mahout at Yahoo\!, May 2008 diff --git a/website/Map_Reduce_Folder/general/faq.md b/website/Map_Reduce_Folder/general/faq.md index 4337f2d7a5..0617834ccc 100644 --- a/website/Map_Reduce_Folder/general/faq.md +++ b/website/Map_Reduce_Folder/general/faq.md @@ -4,7 +4,7 @@ title: FAQ redirect_from: - /general/faq - /general/faq.html - + --- # The Official Mahout FAQ @@ -104,4 +104,4 @@ There is a number of algorithm implementations that require no Hadoop dependenci #### Mahout just won't run in parallel on my dataset. Why? If you are running training on a Hadoop cluster keep in mind that the number of mappers started is governed by the size of the input data and the configured split/block size of your cluster. As a rule of thumb, -anything below 100MB in size won't be split by default. +anything below 100MB in size won't be split by default. diff --git a/website/Map_Reduce_Folder/general/glossary.md b/website/Map_Reduce_Folder/general/glossary.md index 671d60efd9..61cda80cd7 100644 --- a/website/Map_Reduce_Folder/general/glossary.md +++ b/website/Map_Reduce_Folder/general/glossary.md @@ -4,7 +4,7 @@ title: Glossary redirect_from: - /general/glossary - /general/glossary.html - + --- This is a list of common glossary terms used on both the mailing lists and around the site. Where possible I have tried to provide a link to more diff --git a/website/Map_Reduce_Folder/general/mahout-benchmarks.md b/website/Map_Reduce_Folder/general/mahout-benchmarks.md index 293f93363d..ce0e887e68 100644 --- a/website/Map_Reduce_Folder/general/mahout-benchmarks.md +++ b/website/Map_Reduce_Folder/general/mahout-benchmarks.md @@ -150,5 +150,3 @@ measured in seconds, not milliseconds. # Clustering See [MAHOUT-588](https://issues.apache.org/jira/browse/MAHOUT-588) - - diff --git a/website/Map_Reduce_Folder/general/mahout-wiki.md b/website/Map_Reduce_Folder/general/mahout-wiki.md index 5b593b2566..9028e55319 100644 --- a/website/Map_Reduce_Folder/general/mahout-wiki.md +++ b/website/Map_Reduce_Folder/general/mahout-wiki.md @@ -4,10 +4,10 @@ title: Mahout Wiki redirect_from: - /general/mahout-wiki - /general/mahout-wiki.html - + --- Apache Mahout is a new Apache TLP project to create scalable, machine -learning algorithms under the Apache license. +learning algorithms under the Apache license. {toc:style=disc|minlevel=2} @@ -198,4 +198,3 @@ that definitely need to be cleaned up. * {noformat}+*Mahout_(version)*+{noformat} (+*Mahout_0.2*+) is used to draw attention to which version of Mahout a feature was (or will be) added to Mahout. - diff --git a/website/Map_Reduce_Folder/general/powered-by-mahout.md b/website/Map_Reduce_Folder/general/powered-by-mahout.md index 8937884d26..3c0fd4854d 100644 --- a/website/Map_Reduce_Folder/general/powered-by-mahout.md +++ b/website/Map_Reduce_Folder/general/powered-by-mahout.md @@ -4,7 +4,7 @@ title: Powered By Mahout redirect_from: - /general/powered-by-mahout - /general/powered-by-mahout.html - + --- # Powered by Mahout @@ -19,7 +19,7 @@ are for informational purposes only.* ## Commercial Use * Adobe AMP uses Mahout's clustering algorithms to increase video -consumption by better user targeting. +consumption by better user targeting. * Accenture uses Mahout as typical example for their [Hadoop Deployment Comparison Study](http://www.accenture.com/SiteCollectionDocuments/PDF/Accenture-Hadoop-Deployment-Comparison-Study.pdf) * [AOL](http://www.aol.com) use Mahout for shopping recommendations. See [slide deck](http://www.slideshare.net/kryton/the-data-layer) diff --git a/website/Map_Reduce_Folder/general/professional-support.md b/website/Map_Reduce_Folder/general/professional-support.md index f0e14464cf..4d2b9a2f5d 100644 --- a/website/Map_Reduce_Folder/general/professional-support.md +++ b/website/Map_Reduce_Folder/general/professional-support.md @@ -4,7 +4,7 @@ title: Professional Support redirect_from: - /general/professional-support - /general/professional-support.html - + --- diff --git a/website/Map_Reduce_Folder/general/reference-reading.md b/website/Map_Reduce_Folder/general/reference-reading.md index c44d276a6f..6cd70327fe 100644 --- a/website/Map_Reduce_Folder/general/reference-reading.md +++ b/website/Map_Reduce_Folder/general/reference-reading.md @@ -4,7 +4,7 @@ title: Reference Reading redirect_from: - /general/reference-reading - /general/reference-reading.html - + --- # Reference Reading @@ -22,7 +22,7 @@ these are skills that will remain useful decades later. * [Gilbert Strang](http://www-math.mit.edu/~gs) 's [Introduction to Linear Algebra](http://math.mit.edu/linearalgebra/). His [lectures](http://ocw.mit.edu/courses/mathematics/18-06-linear-algebra-spring-2010/) are also [available online](http://web.mit.edu/18.06/www/) - and are strongly recommended. + and are strongly recommended. * [Mathematical Tools for Applied Mulitvariate Analysis](http://www.amazon.com/Mathematical-Tools-Applied-Multivariate-Analysis/dp/0121609553/ref=sr_1_1?ie=UTF8&qid=1299602805&sr=8-1) by J.Douglass Carroll. * [Stanford Machine Learning online courseware](http://www.stanford.edu/class/cs229/) @@ -39,7 +39,7 @@ Some good introductory alternatives here are: Once you have a grasp of the basics then there are a slew of great texts that you might consult: * [Statistical Inference](http://www.amazon.com/Statistical-Inference-George-Casella/dp/0534243126), Casell and Berger, Duxbury/Thomson Learning. - * [Introduction to Bayesian Statistics](http://www.amazon.com/Introduction-Bayesian-Statistics-William-Bolstad/dp/0471270202), William H. Bolstad, Wiley. + * [Introduction to Bayesian Statistics](http://www.amazon.com/Introduction-Bayesian-Statistics-William-Bolstad/dp/0471270202), William H. Bolstad, Wiley. * [Understanding Computational Bayesian Statistics](http://www.amazon.com/Understanding-Computational-Bayesian-Statistics-Wiley/dp/0470046090), Bolstadt * [Bayesian Data Analysis, Gelman et al.](http://www.stat.columbia.edu/~gelman/book/) @@ -47,15 +47,15 @@ Once you have a grasp of the basics then there are a slew of great texts that yo ## For statistics related to machine learning, these are particularly helpful: * [Pattern Recognition and Machine Learning by Chris Bishop](http://research.microsoft.com/en-us/um/people/cmbishop/PRML/index.htm) - * [Elements of Statistical Learning](http://www-stat.stanford.edu/~tibs/ElemStatLearn/) by Trevor Hastie, Robert Tibshirani, Jerome Friedman + * [Elements of Statistical Learning](http://www-stat.stanford.edu/~tibs/ElemStatLearn/) by Trevor Hastie, Robert Tibshirani, Jerome Friedman * [http://research.microsoft.com/en-us/um/people/cmbishop/PRML/index.htm](http://research.microsoft.com/en-us/um/people/cmbishop/PRML/index.htm) - + ## For matrix computations/decomposition/factorization etc.: * Peter V. O'Neil [Introduction to Linear Algebra](http://www.amazon.com/Introduction-Linear-Algebra-Theory-Applications/dp/053400606X), great book for beginners (with some knowledge in calculus). It is not comprehensive, but, it will be a good place to start and the author starts by explaining the concepts with regards to vector spaces which I found to be a more natural way of explaining. * David S. Watkins [Fundamentals of Matrix Computations](http://www.amazon.com/Fundamentals-Matrix-Computations-Applied-Mathematics/dp/0470528338/) - * [Matrix Computations](http://www.amazon.com/Computations-Hopkins-Studies-Mathematical-Sciences/dp/0801854148/ref=sr_1_2?s=books&ie=UTF8&qid=1394307676&sr=1-2&keywords=golub+van+loan) is the classic text for numerical linear algebra. Can't go wrong with it - great for researchers. + * [Matrix Computations](http://www.amazon.com/Computations-Hopkins-Studies-Mathematical-Sciences/dp/0801854148/ref=sr_1_2?s=books&ie=UTF8&qid=1394307676&sr=1-2&keywords=golub+van+loan) is the classic text for numerical linear algebra. Can't go wrong with it - great for researchers. * Nick Trefethen's [Numerical Linear Algebra](http://people.maths.ox.ac.uk/trefethen/books.html). It's a bit more approachable for practitioners. Many chapters on SVD, there are even chapters on Lanczos. @@ -70,4 +70,3 @@ In addition, you should see how to plot data well: * [Trellis plotting](http://www.statmethods.net/advgraphs/trellis.html) * [ggplot2](http://had.co.nz/ggplot2/) - diff --git a/website/Map_Reduce_Folder/general/release-notes.md b/website/Map_Reduce_Folder/general/release-notes.md index 03bdec7f6f..a50f4bf7fe 100644 --- a/website/Map_Reduce_Folder/general/release-notes.md +++ b/website/Map_Reduce_Folder/general/release-notes.md @@ -4,7 +4,7 @@ title: Release Notes redirect_from: - /general/release-notes - /general/release-notes.html - + --- # Release Notes @@ -14,7 +14,7 @@ redirect_from: This release marks a major milestone for the “Samsara” environment’s goal of providing an engine neutral math platform by now supporting Apache Flink. While still experimental, the mahout Flink bindings now offer all of the R-Like -semantics for linear algebra operations, matrix decompositions, +semantics for linear algebra operations, matrix decompositions, and algorithms of the “Samsara” platform for execution on a Flink back-end. This release gives users of Apache Flink out of the box access to the following features (and more): @@ -96,7 +96,7 @@ Highlights include: * Adding in-place elementwise functional assignment (e.g. mxA := exp _, mxA ::= exp _). * Distributed and in-core version of simple elementwise analogues of scala.math._. for example, for log(x) the convention is dlog(drm), mlog(mx), vlog(vec). Unfortunately we cannot overload these functions over what is done in scala.math, i.e. scala would not allow log(mx) or log(drm) and log(Double) at the same time, mainly because they are being defined in different packages. * Distributed and in-core first and second moment routines. R analogs: mean(), colMeans(), rowMeans(), variance(), sd(). By convention, distributed versions are prepended by (d) letter: colMeanVars() colMeanStdevs() dcolMeanVars() dcolMeanStdevs(). -* Distance and squared distance matrix routines. R analog: dist(). Provide both squared and non-squared Euclidean distance matrices. By convention, distributed versions are prepended by (d) letter: dist(x), sqDist(x), dsqDist(x). Also a variation for pair-wise distance matrix of two different inputs x and y: sqDist(x,y), dsqDist(x,y). +* Distance and squared distance matrix routines. R analog: dist(). Provide both squared and non-squared Euclidean distance matrices. By convention, distributed versions are prepended by (d) letter: dist(x), sqDist(x), dsqDist(x). Also a variation for pair-wise distance matrix of two different inputs x and y: sqDist(x,y), dsqDist(x,y). * DRM row sampling api. * Distributed performance bug fixes. This relates mostly to (a) matrix multiplication deficiencies, and (b) handling parallelism. * Distributed engine neutral allreduceBlock() operator api for Spark and H2O. @@ -105,11 +105,11 @@ Highlights include: * Added +=: and *=: operators on vectors. * Closeable API for broadcast tensors. * Support for conversion of any type-keyed DRM into ordinally-keyed DRM. -* Scala logging style. +* Scala logging style. * rowSumsMap() summary for non-int-keyed DRMs. -* elementwise power operator ^ . -* R-like vector concatenation operator. -* In-core functional assignments e.g.: mxA := { (x) => x * x}. +* elementwise power operator ^ . +* R-like vector concatenation operator. +* In-core functional assignments e.g.: mxA := { (x) => x * x}. * Straighten out behavior of Matrix.iterator() and iterateNonEmpty(). * New mutable transposition view for in-core matrices. In-core matrix transpose view. rewrite with mostly two goals in mind: (1) enable mutability, e.g. for (col <- mxA.t) col := k (2) translate matrix structural flavor for optimizers correctly. i.e. new SparseRowMatrix.t carries on as column-major structure. * Native support for kryo serialization of tensor types. @@ -128,10 +128,10 @@ Highlights include: #### 11 April 2015 - Apache Mahout 0.10.0 released -Mahout 0.10.0 was a major release, which separates out a ML environment (we call Mahout-Samsara) including an -extended version of Scala that is largely backend independent but runs fully on Spark. The Hadoop MapReduce versions of -Mahout algorithms are still maintained but no new MapReduce contributions are accepted. From this release onwards -contributions must be Mahout Samsara based or at least run on Spark. +Mahout 0.10.0 was a major release, which separates out a ML environment (we call Mahout-Samsara) including an +extended version of Scala that is largely backend independent but runs fully on Spark. The Hadoop MapReduce versions of +Mahout algorithms are still maintained but no new MapReduce contributions are accepted. From this release onwards +contributions must be Mahout Samsara based or at least run on Spark. Highlights include: @@ -240,5 +240,5 @@ Changes in 0.10.0 are detailed ## Unit Tests @@ -45,4 +45,3 @@ of a great many utilities and algorithms. A much smaller percent include ## Other - diff --git a/website/Map_Reduce_Folder/misc/using-mahout-with-python-via-jpype.md b/website/Map_Reduce_Folder/misc/using-mahout-with-python-via-jpype.md index 9b51c63d48..0d6f7cea42 100644 --- a/website/Map_Reduce_Folder/misc/using-mahout-with-python-via-jpype.md +++ b/website/Map_Reduce_Folder/misc/using-mahout-with-python-via-jpype.md @@ -65,20 +65,20 @@ vectors from two Gaussian distributions with unit variance. """Create a sequence file containing some normally distributed ifile - path to the sequence file to create """ - + #matrix of the cluster means cmeans=np.array([[1,1] ,[-1,-1]],np.int) - + nperc=30 #number of points per cluster - + vecs=[] - + vnames=[] for cind in range(cmeans.shape[0]): pts=np.random.randn(nperc,2) pts=pts+cmeans[cind,:].reshape([1,cmeans.shape[1]]) vecs.append(pts) - + #names for the vectors #names are just the points with an index #we do this so we can validate by cross-refencing the name with thevector @@ -86,43 +86,43 @@ vectors from two Gaussian distributions with unit variance. for row in range(nperc): vn[row]="c"+str(cind)+"_"+pts[row,0].astype((np.str,4))+"_"+pts[row,1].astype((np.str,4)) vnames.append(vn) - + vecs=np.vstack(vecs) vnames=np.hstack(vnames) - - + + #start the jvm start_jpype() - + #create the sequence file that we will write to - io=JPackage("org").apache.hadoop.io + io=JPackage("org").apache.hadoop.io FileSystemCls=JPackage("org").apache.hadoop.fs.FileSystem - + PathCls=JPackage("org").apache.hadoop.fs.Path path=PathCls(ifile) - - ConfCls=JPackage("org").apache.hadoop.conf.Configuration + + ConfCls=JPackage("org").apache.hadoop.conf.Configuration conf=ConfCls() - + fs=FileSystemCls.get(conf) - + #vector classes VectorWritableCls=JPackage("org").apache.mahout.math.VectorWritable DenseVectorCls=JPackage("org").apache.mahout.math.DenseVector NamedVectorCls=JPackage("org").apache.mahout.math.NamedVector writer=io.SequenceFile.createWriter(fs, conf, path,io.Text,VectorWritableCls) - - + + vecwritable=VectorWritableCls() for row in range(vecs.shape[0]): nvector=NamedVectorCls(DenseVectorCls(JArray(JDouble,1)(vecs[row,:])),vnames[row]) #need to wrap key and value because of overloading wrapkey=JObject(io.Text("key "+str(row)),io.Writable) wrapval=JObject(vecwritable,io.Writable) - + vecwritable.set(nvector) writer.append(wrapkey,wrapval) - + writer.close() @@ -134,43 +134,43 @@ mahout. def read_clustered_pts(ifile,*args,**param): """Read the clustered points ifile - path to the sequence file containing the clustered points - """ - + """ + #start the jvm start_jpype() - + #create the sequence file that we will write to - io=JPackage("org").apache.hadoop.io + io=JPackage("org").apache.hadoop.io FileSystemCls=JPackage("org").apache.hadoop.fs.FileSystem - + PathCls=JPackage("org").apache.hadoop.fs.Path path=PathCls(ifile) - - ConfCls=JPackage("org").apache.hadoop.conf.Configuration + + ConfCls=JPackage("org").apache.hadoop.conf.Configuration conf=ConfCls() - + fs=FileSystemCls.get(conf) - + #vector classes VectorWritableCls=JPackage("org").apache.mahout.math.VectorWritable NamedVectorCls=JPackage("org").apache.mahout.math.NamedVector - - - ReaderCls=io.__getattribute__("SequenceFile$Reader") + + + ReaderCls=io.__getattribute__("SequenceFile$Reader") reader=ReaderCls(fs, path,conf) - - + + key=reader.getKeyClass()() - - + + valcls=reader.getValueClass() vecwritable=valcls() - while (reader.next(key,vecwritable)): + while (reader.next(key,vecwritable)): weight=vecwritable.getWeight() nvec=vecwritable.getVector() - + cname=nvec.__class__.__name__ - if (cname.rsplit('.',1)[1]=="NamedVector"): + if (cname.rsplit('.',1)[1]=="NamedVector"): print "cluster={key} Name={name} x={x}y={y}".format(key=key.toString(),name=nvec.getName(),x=nvec.get(0),y=nvec.get(1)) else: raise NotImplementedError("Vector isn't a NamedVector. Need tomodify/test the code to handle this case.") @@ -184,40 +184,39 @@ found by mahout, def getClusters(ifile,*args,**param): """Read the centroids from the clusters outputted by kmenas ifile - Path to the sequence file containing the centroids - """ - + """ + #start the jvm start_jpype() - + #create the sequence file that we will write to - io=JPackage("org").apache.hadoop.io + io=JPackage("org").apache.hadoop.io FileSystemCls=JPackage("org").apache.hadoop.fs.FileSystem - + PathCls=JPackage("org").apache.hadoop.fs.Path path=PathCls(ifile) - - ConfCls=JPackage("org").apache.hadoop.conf.Configuration + + ConfCls=JPackage("org").apache.hadoop.conf.Configuration conf=ConfCls() - + fs=FileSystemCls.get(conf) - + #vector classes VectorWritableCls=JPackage("org").apache.mahout.math.VectorWritable NamedVectorCls=JPackage("org").apache.mahout.math.NamedVector ReaderCls=io.__getattribute__("SequenceFile$Reader") reader=ReaderCls(fs, path,conf) - - + + key=io.Text() - - + + valcls=reader.getValueClass() - + vecwritable=valcls() - - while (reader.next(key,vecwritable)): + + while (reader.next(key,vecwritable)): center=vecwritable.getCenter() - + print "id={cid}center={center}".format(cid=vecwritable.getId(),center=center.values) pass - diff --git a/website/Map_Reduce_Folder/news-and-events/events.md b/website/Map_Reduce_Folder/news-and-events/events.md index 588f022010..40103c85a0 100644 --- a/website/Map_Reduce_Folder/news-and-events/events.md +++ b/website/Map_Reduce_Folder/news-and-events/events.md @@ -17,6 +17,3 @@ Check out the upcoming events related to the Apache Mahout project! Join webinar | 2023-05-31 | Mahout Community Call | Online | A Monthly Community Call Where project leaders discuss pressing issues | Don't forget to check back frequently for updates on new events and opportunities to connect with the Mahout community. We look forward to seeing you at an event soon! - - - diff --git a/website/Map_Reduce_Folder/news-and-events/news.md b/website/Map_Reduce_Folder/news-and-events/news.md index a921c8a5bb..3db36a3c6c 100644 --- a/website/Map_Reduce_Folder/news-and-events/news.md +++ b/website/Map_Reduce_Folder/news-and-events/news.md @@ -9,10 +9,9 @@ redirect_from: # News Welcome to the News page! Stay up-to-date with the latest announcements, releases, events, and community highlights from -the Apache Mahout project. Keep an eye on this page for regular updates and make sure you don't miss any important news +the Apache Mahout project. Keep an eye on this page for regular updates and make sure you don't miss any important news related to the project. {% for post in site.posts limit:10 %} - [{{post.title}}]({{ post.url }}) - {{ post.date | date: "%B %d, %Y" }} {% endfor %} - diff --git a/website/Map_Reduce_Folder/recommender/intro-als-hadoop.md b/website/Map_Reduce_Folder/recommender/intro-als-hadoop.md index 1fff25b7a8..426c9af3ce 100644 --- a/website/Map_Reduce_Folder/recommender/intro-als-hadoop.md +++ b/website/Map_Reduce_Folder/recommender/intro-als-hadoop.md @@ -10,11 +10,11 @@ redirect_from: ##Overview -Mahout’s ALS recommender is a matrix factorization algorithm that uses Alternating Least Squares with Weighted-Lamda-Regularization (ALS-WR). It factors the user to item matrix *A* into the user-to-feature matrix *U* and the item-to-feature matrix *M*: It runs the ALS algorithm in a parallel fashion. The algorithm details can be referred to in the following papers: +Mahout’s ALS recommender is a matrix factorization algorithm that uses Alternating Least Squares with Weighted-Lamda-Regularization (ALS-WR). It factors the user to item matrix *A* into the user-to-feature matrix *U* and the item-to-feature matrix *M*: It runs the ALS algorithm in a parallel fashion. The algorithm details can be referred to in the following papers: * [Large-scale Parallel Collaborative Filtering for the Netflix Prize](http://www.hpl.hp.com/personal/Robert_Schreiber/papers/2008%20AAIM%20Netflix/netflix_aaim08%28submitted%29.pdf) -* [Collaborative Filtering for Implicit Feedback Datasets](http://research.yahoo.com/pub/2433) +* [Collaborative Filtering for Implicit Feedback Datasets](http://research.yahoo.com/pub/2433) This recommendation algorithm can be used in eCommerce platform to recommend products to customers. Unlike the user or item based recommenders that computes the similarity of users or items to make recommendations, the ALS algorithm uncovers the latent factors that explain the observed user to item ratings and tries to find optimal factor weights to minimize the least squares between predicted and actual ratings. @@ -24,7 +24,7 @@ One of the strengths of the ALS based recommender, compared to the user or item ##Implementation At present Mahout has a map-reduce implementation of ALS, which is composed of 2 jobs: a parallel matrix factorization job and a recommendation job. -The matrix factorization job computes the user-to-feature matrix and item-to-feature matrix given the user to item ratings. Its input includes: +The matrix factorization job computes the user-to-feature matrix and item-to-feature matrix given the user to item ratings. Its input includes:
     --input: directory containing files of explicit user to item rating or implicit feedback;
     --output: output path of the user-feature matrix and feature-item matrix;
@@ -36,7 +36,7 @@ The matrix factorization job computes the user-to-feature matrix and item-to-fea
     --numIterations: number of iterations
     --usesLongIDs: boolean flag to indicate whether the input contains long IDs that need to be translated
 
-and it outputs the matrices in sequence file format. +and it outputs the matrices in sequence file format. The recommendation job uses the user feature matrix and item feature matrix calculated from the factorization job to compute the top-N recommendations per user. Its input includes:
@@ -49,9 +49,9 @@ The recommendation job uses the user feature matrix and item feature matrix calc
     --numThreads: number of threads per mapper;
     --usesLongIDs: boolean flag to indicate whether the input contains long IDs that need to be translated;
     --userIDIndex: index for user long IDs (necessary if usesLongIDs is true);
-    --itemIDIndex: index for item long IDs (necessary if usesLongIDs is true) 
+    --itemIDIndex: index for item long IDs (necessary if usesLongIDs is true)
 
-and it outputs a list of recommended item ids for each user. The predicted rating between user and item is a dot product of the user's feature vector and the item's feature vector. +and it outputs a list of recommended item ids for each user. The predicted rating between user and item is a dot product of the user's feature vector and the item's feature vector. ##Example @@ -76,13 +76,13 @@ To quickly start, you could specify a text file like following as the input: **Step 2: Determine parameters** -In addition, users need to determine dimension of feature space, the number of iterations to run the alternating least square algorithm, Using 10 features and 15 iterations is a reasonable default to try first. Optionally a confidence parameter can be set if the input preference is implicit user feedback. +In addition, users need to determine dimension of feature space, the number of iterations to run the alternating least square algorithm, Using 10 features and 15 iterations is a reasonable default to try first. Optionally a confidence parameter can be set if the input preference is implicit user feedback. **Step 3: Run ALS** Assuming your *JAVA_HOME* is appropriately set and Mahout was installed properly we’re ready to configure our syntax. Enter the following command: - $ mahout parallelALS --input $als_input --output $als_output --lambda 0.1 --implicitFeedback true --alpha 0.8 --numFeatures 2 --numIterations 5 --numThreadsPerSolver 1 --tempDir tmp + $ mahout parallelALS --input $als_input --output $als_output --lambda 0.1 --implicitFeedback true --alpha 0.8 --numFeatures 2 --numIterations 5 --numThreadsPerSolver 1 --tempDir tmp Running the command will execute a series of jobs the final product of which will be an output file deposited to the output directory specified in the command syntax. The output directory contains 3 sub-directories: *M* stores the item to feature matrix, *U* stores the user to feature matrix and userRatings stores the user's ratings on the items. The *tempDir* parameter specifies the directory to store the intermediate output of the job, such as the matrix output in each iteration and each item's average rating. Using the *tempDir* will help on debugging. @@ -92,8 +92,7 @@ Based on the output feature matrices from step 3, we could make recommendations $ mahout recommendfactorized --input $als_recommender_input --userFeatures $als_output/U/ --itemFeatures $als_output/M/ --numRecommendations 1 --output recommendations --maxRating 1 -The input user file is a sequence file, the sequence record key is user id and value is the user's rated item ids which will be removed from recommendation. The output file generated in our simple example will be a text file giving the recommended item ids for each user. -Remember to translate the Mahout ids back into your application specific ids. +The input user file is a sequence file, the sequence record key is user id and value is the user's rated item ids which will be removed from recommendation. The output file generated in our simple example will be a text file giving the recommended item ids for each user. +Remember to translate the Mahout ids back into your application specific ids. There exist a variety of parameters for Mahout’s ALS recommender to accommodate custom business requirements; exploring and testing various configurations to suit your needs will doubtless lead to additional questions. Feel free to ask such questions on the [mailing list](https://mahout.apache.org/community/mailing-lists.html). - diff --git a/website/Map_Reduce_Folder/recommender/intro-cooccurrence-spark.md b/website/Map_Reduce_Folder/recommender/intro-cooccurrence-spark.md index aa5469b5d0..286598dda0 100644 --- a/website/Map_Reduce_Folder/recommender/intro-cooccurrence-spark.md +++ b/website/Map_Reduce_Folder/recommender/intro-cooccurrence-spark.md @@ -8,7 +8,7 @@ redirect_from: # Building a Correlated Cross-Occurrence (CCO) Recommenders with the Mahout CLI -Mahout's CCO algorithm is one of a new breed of "Multimodal" recommenders that can use input of many types in very flexible ways. +Mahout's CCO algorithm is one of a new breed of "Multimodal" recommenders that can use input of many types in very flexible ways. Mahout provides several important building blocks for creating recommendations using Spark. *spark-itemsimilarity* can be used to create "other people also liked these things" type recommendations and paired with a search engine can personalize recommendations for individual users. *spark-rowsimilarity* can provide non-personalized content based recommendations and when paired with a search engine can be used to personalize content based recommendations. @@ -16,7 +16,7 @@ Mahout provides several important building blocks for creating recommendations u This is a simplified Lambda architecture with Mahout's *spark-itemsimilarity* playing the batch model building role and a search engine playing the realtime serving role. -You will create two collections, one for user history and one for item "indicators". Indicators are user interactions that lead to the wished for interaction. So for example if you wish a user to purchase something and you collect all users purchase interactions *spark-itemsimilarity* will create a purchase indicator from them. But you can also use other user interactions in a cross-cooccurrence calculation, to create purchase indicators. +You will create two collections, one for user history and one for item "indicators". Indicators are user interactions that lead to the wished for interaction. So for example if you wish a user to purchase something and you collect all users purchase interactions *spark-itemsimilarity* will create a purchase indicator from them. But you can also use other user interactions in a cross-cooccurrence calculation, to create purchase indicators. User history is used as a query on the item collection with its cooccurrence and cross-cooccurrence indicators (there may be several indicators). The primary interaction or indicator is picked to be the thing you want to recommend, other action / indicators are believed to be correlated but may not indicate exactly the same user intent. For instance in an ecom recommender a purchase is a very good primary action / indicator, but you may also know product detail-views, or additions-to-wishlists. These can be considered secondary actions / indicators which may all be used to calculate cross-cooccurrence indicators. The user history that forms the recommendations query will contain recorded primary and secondary indicators all targeted towards the correct indicator fields. @@ -33,20 +33,20 @@ Below are the command line jobs but the drivers and associated code can also be ## 1. spark-itemsimilarity *spark-itemsimilarity* is the Spark counterpart of the of the Mahout mapreduce job called *itemsimilarity*. It takes in elements of interactions, which have userID, itemID, and optionally a value. It will produce one of more indicator matrices created by comparing every user's interactions with every other user. The indicator matrix is an item x item matrix where the values are log-likelihood ratio strengths. For the legacy mapreduce version, there were several possible similarity measures but these are being deprecated in favor of LLR because in practice it performs the best. -Mahout's mapreduce version of itemsimilarity takes a text file that is expected to have user and item IDs that conform to +Mahout's mapreduce version of itemsimilarity takes a text file that is expected to have user and item IDs that conform to Mahout's ID requirements--they are non-negative integers that can be viewed as row and column numbers in a matrix. -*spark-itemsimilarity* also extends the notion of cooccurrence to cross-cooccurrence, in other words the Spark version will -account for multi-modal interactions and create cross-cooccurrence indicator matrices allowing the use of much more data in -creating recommendations or similar item lists. People try to do this by mixing different indicators and giving them weights. +*spark-itemsimilarity* also extends the notion of cooccurrence to cross-cooccurrence, in other words the Spark version will +account for multi-modal interactions and create cross-cooccurrence indicator matrices allowing the use of much more data in +creating recommendations or similar item lists. People try to do this by mixing different indicators and giving them weights. For instance they might say an item-view is 0.2 of an item purchase. In practice this is often not helpful. Spark-itemsimilarity's cross-cooccurrence is a more principled way to handle this case. In effect it scrubs secondary indicators with the indicator you want -to recommend. +to recommend. spark-itemsimilarity Mahout 1.0 Usage: spark-itemsimilarity [options] - + Disconnected from the target VM, address: '127.0.0.1:64676', transport: 'socket' Input, output options -i | --input @@ -55,15 +55,15 @@ to recommend. Secondary input path for cross-similarity calculation, same restrictions as "--input" (optional). Default: empty. -o | --output Path for output, any local or HDFS supported URI (required) - + Algorithm control options: -mppu | --maxPrefs Max number of preferences to consider per user (optional). Default: 500 -m | --maxSimilaritiesPerItem Limit the number of similarities per item to this number (optional). Default: 100 - + Note: Only the Log Likelihood Ratio (LLR) is supported as a similarity measure. - + Input text file schema options: -id | --inDelim Input delimiter character (optional). Default: "[,\t]" @@ -77,15 +77,15 @@ to recommend. Column number (0 based Int) containing the item ID string (optional). Default: 1 -fc | --filterColumn Column number (0 based Int) containing the filter string (optional). Default: -1 for no filter - + Using all defaults the input is expected of the form: "userIDitemId" or "userIDitemIDany-text..." and all rows will be used - + File discovery options: -r | --recursive Searched the -i path recursively for files that match --filenamePattern (optional), Default: false -fp | --filenamePattern Regex to match in determining input files (optional). Default: filename in the --input option or "^part-.*" if --input is a directory - + Output text file schema options: -rd | --rowKeyDelim Separates the rowID key from the vector values list (optional). Default: "\t" @@ -96,22 +96,22 @@ to recommend. -os | --omitStrength Do not write the strength to the output files (optional), Default: false. This option is used to output indexable data for creating a search engine recommender. - + Default delimiters will produce output of the form: "itemID1itemID2:value2itemID10:value10..." - + Spark config options: -ma | --master Spark Master URL (optional). Default: "local". Note that you can specify the number of cores to get a performance improvement, for example "local[4]" -sem | --sparkExecutorMem Max Java heap available as "executor memory" on each node (optional). Default: 4g -rs | --randomSeed - + -h | --help prints this usage text This looks daunting but defaults to simple fairly sane values to take exactly the same input as legacy code and is pretty flexible. It allows the user to point to a single text file, a directory full of files, or a tree of directories to be traversed recursively. The files included can be specified with either a regex-style pattern or filename. The schema for the file is defined by column numbers, which map to the important bits of data including IDs and values. The files can even contain filters, which allow unneeded rows to be discarded or used for cross-cooccurrence calculations. -See `ItemSimilarityDriver.scala` in Mahout's spark module if you want to customize the code. +See `ItemSimilarityDriver.scala` in Mahout's spark module if you want to customize the code. ### Defaults in the _**spark-itemsimilarity**_ CLI @@ -133,13 +133,13 @@ This will use the "local" Spark context and will output the standard text versio ### How To Use Multiple User Indicators -Often we record various indicators the user takes for later analytics. These can now be used to make recommendations. +Often we record various indicators the user takes for later analytics. These can now be used to make recommendations. The idea of a recommender is to recommend the action you want the user to make. For an ecom app this might be a purchase action recorded in a "purchase" indicator. It is usually not a good idea to just treat other indicators the same as the indicator you want to recommend. For example is you have user purchase and view data, never treat a view as a purchase it will never increase the quality of recommendations, instead use the view data as a secondary indicator so the CCO algorithm will find meaningful correlated cross-occurrences. Without this the views will be so noisy they will almost surely reduce the performance of the recommender. Too many people have fallen into this mistake. With *spark-itemsimilarity* we can now use both indicators. Mahout will use cross-occurrence analysis to limit the views to ones that do predict purchases. -We do this by treating the primary indicator (purchase) as data for the indicator matrix and use the secondary indicator (view) -to calculate the cross-cooccurrence indicator matrix. +We do this by treating the primary indicator (purchase) as data for the indicator matrix and use the secondary indicator (view) +to calculate the cross-cooccurrence indicator matrix. -*spark-itemsimilarity* can read separate indicators from separate files or from a mixed indicator log by filtering certain lines. For a mixed +*spark-itemsimilarity* can read separate indicators from separate files or from a mixed indicator log by filtering certain lines. For a mixed indicator log of the form: u1,purchase,iphone @@ -182,7 +182,7 @@ Use the following options: ### Output -The output of the job will be the standard text version of two Mahout DRMs. This is a case where we are calculating +The output of the job will be the standard text version of two Mahout DRMs. This is a case where we are calculating cross-cooccurrence so a primary indicator matrix and cross-cooccurrence indicator matrix will be created out-path @@ -230,7 +230,7 @@ A common method of storing data is in log files. If they are written using some 2014-06-23 14:46:53.115u3viewrandom textnexus 2014-06-23 14:46:53.115u4viewrandom textiphone 2014-06-23 14:46:53.115u4viewrandom textipad - 2014-06-23 14:46:53.115u4viewrandom textgalaxy + 2014-06-23 14:46:53.115u4viewrandom textgalaxy Can be parsed with the following CLI and run on the cluster producing the same output as the above example. The important bit of information in the example tab delimited file are user-id, indicator-name, and item-id. The rest is ignored. @@ -247,35 +247,35 @@ Can be parsed with the following CLI and run on the cluster producing the same o ## 2. spark-rowsimilarity -*spark-rowsimilarity* is the companion to *spark-itemsimilarity* the primary difference is that it takes a text file version of +*spark-rowsimilarity* is the companion to *spark-itemsimilarity* the primary difference is that it takes a text file version of a matrix of sparse vectors with optional application specific IDs and it finds similar rows rather than items (columns). Its use is -not limited to collaborative filtering. The input is in text-delimited form where there are three delimiters used. By +not limited to collaborative filtering. The input is in text-delimited form where there are three delimiters used. By default it reads `(rowIDcolumnID1:strength1columnID2:strength2...)` Since this job only supports LLR similarity, - which does not use the input strengths, they may be omitted in the input. It writes -`(rowIDrowID1:strength1rowID2:strength2...)` -The output is sorted by strength descending. The output can be interpreted as a row ID from the primary input followed + which does not use the input strengths, they may be omitted in the input. It writes +`(rowIDrowID1:strength1rowID2:strength2...)` +The output is sorted by strength descending. The output can be interpreted as a row ID from the primary input followed by a list of the most similar rows. The command line interface is: spark-rowsimilarity Mahout 0.x Usage: spark-rowsimilarity [options] - + Input, output options -i | --input Input path, may be a filename, directory name, or comma delimited list of HDFS supported URIs (required) -o | --output Path for output, any local or HDFS supported URI (required) - + Algorithm control options: -mo | --maxObservations Max number of observations to consider per row (optional). Default: 500 -m | --maxSimilaritiesPerRow Limit the number of similarities per item to this number (optional). Default: 100 - + Note: Only the Log Likelihood Ratio (LLR) is supported as a similarity measure. Disconnected from the target VM, address: '127.0.0.1:49162', transport: 'socket' - + Output text file schema options: -rd | --rowKeyDelim Separates the rowID key from the vector values list (optional). Default: "\t" @@ -286,39 +286,39 @@ The command line interface is: -os | --omitStrength Do not write the strength to the output files (optional), Default: false. This option is used to output indexable data for creating a search engine recommender. - + Default delimiters will produce output of the form: "itemID1itemID2:value2itemID10:value10..." - + File discovery options: -r | --recursive Searched the -i path recursively for files that match --filenamePattern (optional), Default: false -fp | --filenamePattern Regex to match in determining input files (optional). Default: filename in the --input option or "^part-.*" if --input is a directory - + Spark config options: -ma | --master Spark Master URL (optional). Default: "local". Note that you can specify the number of cores to get a performance improvement, for example "local[4]" -sem | --sparkExecutorMem Max Java heap available as "executor memory" on each node (optional). Default: 4g -rs | --randomSeed - + -h | --help prints this usage text -See RowSimilarityDriver.scala in Mahout's spark module if you want to customize the code. +See RowSimilarityDriver.scala in Mahout's spark module if you want to customize the code. #3. Using *spark-rowsimilarity* with Text Data -Another use case for *spark-rowsimilarity* is in finding similar textual content. For instance given the tags associated with -a blog post, which other posts have similar tags. In this case the columns are tags and the rows are posts. Since LLR is -the only similarity method supported this is not the optimal way to determine general "bag-of-words" document similarity. -LLR is used more as a quality filter than as a similarity measure. However *spark-rowsimilarity* will produce +Another use case for *spark-rowsimilarity* is in finding similar textual content. For instance given the tags associated with +a blog post, which other posts have similar tags. In this case the columns are tags and the rows are posts. Since LLR is +the only similarity method supported this is not the optimal way to determine general "bag-of-words" document similarity. +LLR is used more as a quality filter than as a similarity measure. However *spark-rowsimilarity* will produce lists of similar docs for every doc if input is docs with lists of terms. The Apache [Lucene](http://lucene.apache.org) project provides several methods of analyzing and tokenizing documents. # 4. Creating a Multimodal Recommender Using the output of *spark-itemsimilarity* and *spark-rowsimilarity* you can build a miltimodal cooccurrence and content based - recommender that can be used in both or either mode depending on indicators available and the history available at + recommender that can be used in both or either mode depending on indicators available and the history available at runtime for a user. Some slide describing this method can be found [here](http://occamsmachete.com/ml/2014/10/07/creating-a-unified-recommender-with-mahout-and-a-search-engine/) ## Requirements @@ -336,43 +336,43 @@ Indicators come in 3 types 2. **Content**: calculated from item metadata or content using *spark-rowsimilarity* 3. **Intrinsic**: assigned to items as metadata. Can be anything that describes the item. These will be used in search engine queries to implement business rules. -The query for recommendations will be a mix of values meant to match one of your indicators. The query can be constructed -from user history and values derived from context (category being viewed for instance) or special pre-calculated data -(popularity rank for instance). This blending of indicators allows for creating many flavors or recommendations to fit +The query for recommendations will be a mix of values meant to match one of your indicators. The query can be constructed +from user history and values derived from context (category being viewed for instance) or special pre-calculated data +(popularity rank for instance). This blending of indicators allows for creating many flavors or recommendations to fit a very wide variety of circumstances. -With the right mix of indicators developers can construct a single query that works for completely new items and new users +With the right mix of indicators developers can construct a single query that works for completely new items and new users while working well for items with lots of interactions and users with many recorded indicators. In other words by adding in content and intrinsic indicators developers can create a solution for the "cold-start" problem that gracefully improves with more user history -and as items have more interactions. It is also possible to create a completely content-based recommender that personalizes +and as items have more interactions. It is also possible to create a completely content-based recommender that personalizes recommendations. ## Example with 3 Indicators -You will need to decide how you store user indicator data so they can be processed by the item and row similarity jobs and -this is most easily done by using text files as described above. The data that is processed by these jobs is considered the -training data. You will need some amount of user history in your recs query. It is typical to use the most recent user history -but need not be exactly what is in the training set, which may include a greater volume of historical data. Keeping the user -history for query purposes could be done with a database by storing it in a users table. In the example above the two -collaborative filtering indicators are "purchase" and "view", but let's also add tags (taken from catalog categories or other -descriptive metadata). +You will need to decide how you store user indicator data so they can be processed by the item and row similarity jobs and +this is most easily done by using text files as described above. The data that is processed by these jobs is considered the +training data. You will need some amount of user history in your recs query. It is typical to use the most recent user history +but need not be exactly what is in the training set, which may include a greater volume of historical data. Keeping the user +history for query purposes could be done with a database by storing it in a users table. In the example above the two +collaborative filtering indicators are "purchase" and "view", but let's also add tags (taken from catalog categories or other +descriptive metadata). -We will need to create 1 cooccurrence indicator from the primary indicator (purchase) 1 cross-occurrence indicator -from the secondary indicator (view) +We will need to create 1 cooccurrence indicator from the primary indicator (purchase) 1 cross-occurrence indicator +from the secondary indicator (view) and 1 content indicator (tags). We'll have to run *spark-itemsimilarity* once and *spark-rowsimilarity* once. -We have described how to create the collaborative filtering indicators for purchase and view (the [How to use Multiple User -Indicators](#multiple-actions) section) but tags will be a slightly different process. We want to use the fact that -certain items have tags similar to the ones associated with a user's purchases. This is not a collaborative filtering indicator -but rather a "content" or "metadata" type indicator since you are not using other users' history, only the -individual that you are making recs for. This means that this method will make recommendations for items that have +We have described how to create the collaborative filtering indicators for purchase and view (the [How to use Multiple User +Indicators](#multiple-actions) section) but tags will be a slightly different process. We want to use the fact that +certain items have tags similar to the ones associated with a user's purchases. This is not a collaborative filtering indicator +but rather a "content" or "metadata" type indicator since you are not using other users' history, only the +individual that you are making recs for. This means that this method will make recommendations for items that have no collaborative filtering data, as happens with new items in a catalog. New items may have tags assigned but no one has purchased or viewed them yet. In the final query we will mix all 3 indicators. ## Content Indicator -To create a content-indicator we'll make use of the fact that the user has purchased items with certain tags. We want to find -items with the most similar tags. Notice that other users' behavior is not considered--only other item's tags. This defines a -content or metadata indicator. They are used when you want to find items that are similar to other items by using their +To create a content-indicator we'll make use of the fact that the user has purchased items with certain tags. We want to find +items with the most similar tags. Notice that other users' behavior is not considered--only other item's tags. This defines a +content or metadata indicator. They are used when you want to find items that are similar to other items by using their content or metadata, not by which users interacted with them. **Note**: It may be advisable to treat tags as cross-cooccurrence indicators but for the sake of an example they are treated here as content only. @@ -388,9 +388,9 @@ The full collection will look like the tags column from a catalog DB. For our ec 9446577dwomen tops chambray clothing casual ... -We'll use *spark-rowimilairity* because we are looking for similar rows, which encode items in this case. As with the -collaborative filtering indicators we use the --omitStrength option. The strengths created are -probabilistic log-likelihood ratios and so are used to filter unimportant similarities. Once the filtering or downsampling +We'll use *spark-rowimilairity* because we are looking for similar rows, which encode items in this case. As with the +collaborative filtering indicators we use the --omitStrength option. The strengths created are +probabilistic log-likelihood ratios and so are used to filter unimportant similarities. Once the filtering or downsampling is finished we no longer need the strengths. We will get an indicator matrix of the form: itemIDlist-of-item IDs @@ -400,30 +400,30 @@ This is a content indicator since it has found other items with similar content 3459860b3459860b 3459860b 6749860c 5959860a 3434860a 3477860a 9446577d9446577d 9496577d 0943577d 8346577d 9442277d 9446577e - ... - + ... + We now have three indicators, two collaborative filtering type and one content type. ## Multimodal Recommender Query -The actual form of the query for recommendations will vary depending on your search engine but the intent is the same. For a given user, map their history of an indicator or content to the correct indicator field and perform an OR'd query. +The actual form of the query for recommendations will vary depending on your search engine but the intent is the same. For a given user, map their history of an indicator or content to the correct indicator field and perform an OR'd query. -We have 3 indicators, these are indexed by the search engine into 3 fields, we'll call them "purchase", "view", and "tags". +We have 3 indicators, these are indexed by the search engine into 3 fields, we'll call them "purchase", "view", and "tags". We take the user's history that corresponds to each indicator and create a query of the form: Query: field: purchase; q:user's-purchase-history field: view; q:user's view-history field: tags; q:user's-tags-associated-with-purchases - -The query will result in an ordered list of items recommended for purchase but skewed towards items with similar tags to -the ones the user has already purchased. -This is only an example and not necessarily the optimal way to create recs. It illustrates how business rules can be -translated into recommendations. This technique can be used to skew recommendations towards intrinsic indicators also. -For instance you may want to put personalized popular item recs in a special place in the UI. Create a popularity indicator +The query will result in an ordered list of items recommended for purchase but skewed towards items with similar tags to +the ones the user has already purchased. + +This is only an example and not necessarily the optimal way to create recs. It illustrates how business rules can be +translated into recommendations. This technique can be used to skew recommendations towards intrinsic indicators also. +For instance you may want to put personalized popular item recs in a special place in the UI. Create a popularity indicator by tagging items with some category of popularity (hot, warm, cold for instance) then -index that as a new indicator field and include the corresponding value in a query +index that as a new indicator field and include the corresponding value in a query on the popularity field. If we use the ecom example but use the query to get "hot" recommendations it might look like this: Query: diff --git a/website/Map_Reduce_Folder/recommender/intro-itembased-hadoop.md b/website/Map_Reduce_Folder/recommender/intro-itembased-hadoop.md index 7467dba5df..9fe837333d 100644 --- a/website/Map_Reduce_Folder/recommender/intro-itembased-hadoop.md +++ b/website/Map_Reduce_Folder/recommender/intro-itembased-hadoop.md @@ -53,4 +53,4 @@ Running the command will execute a series of jobs the final product of which wil **Step 4: Making use of the output and doing more with Mahout** -The output file generated in our simple example can be transformed using your tool of choice and consumed by downstream applications. There exist a variety of configuration options for Mahout’s item based recommender to accommodate custom business requirements; exploring and testing various configurations to suit your needs will doubtless lead to additional questions. Our user community is accessible via our [mailing list](https://mahout.apache.org/community/mailing-lists.html) and the book *Mahout In Action* is a fantastic (but slightly outdated) starting point. +The output file generated in our simple example can be transformed using your tool of choice and consumed by downstream applications. There exist a variety of configuration options for Mahout’s item based recommender to accommodate custom business requirements; exploring and testing various configurations to suit your needs will doubtless lead to additional questions. Our user community is accessible via our [mailing list](https://mahout.apache.org/community/mailing-lists.html) and the book *Mahout In Action* is a fantastic (but slightly outdated) starting point. diff --git a/website/Map_Reduce_Folder/recommender/matrix-factorization.md b/website/Map_Reduce_Folder/recommender/matrix-factorization.md index bbcbabd6bc..bb5325cb47 100644 --- a/website/Map_Reduce_Folder/recommender/matrix-factorization.md +++ b/website/Map_Reduce_Folder/recommender/matrix-factorization.md @@ -9,14 +9,14 @@ redirect_from: # Introduction to Matrix Factorization for Recommendation Mining -In the mathematical discipline of linear algebra, a matrix decomposition -or matrix factorization is a dimensionality reduction technique that factorizes a matrix into a product of matrices, usually two. +In the mathematical discipline of linear algebra, a matrix decomposition +or matrix factorization is a dimensionality reduction technique that factorizes a matrix into a product of matrices, usually two. There are many different matrix decompositions, each finds use among a particular class of problems. In mahout, the SVDRecommender provides an interface to build recommender based on matrix factorization. The idea behind is to project the users and items onto a feature space and try to optimize U and M so that U \* (M^t) is as close to R as possible: - U is n * p user feature matrix, + U is n * p user feature matrix, M is m * p item feature matrix, M^t is the conjugate transpose of M, R is n * m rating matrix, n is the number of users, @@ -28,14 +28,14 @@ RMSE is defined as the squared root of the sum of squared errors at each known u So our matrix factorization target could be mathmatically defined as: find U and M, (U, M) = argmin(RMSE) = argmin(pow(SSE / K, 0.5)) - + SSE = sum(e(u,i)^2) e(u,i) = r(u, i) - U[u,] * (M[i,]^t) = r(u,i) - sum(U[u,f] * M[i,f]), f = 0, 1, .. p - 1 K is the number of known user item ratings. -Mahout has implemented matrix factorization based on +Mahout has implemented matrix factorization based on (1) SGD(Stochastic Gradient Descent) (2) ALSWR(Alternating-Least-Squares with Weighted-λ-Regularization). @@ -44,11 +44,11 @@ Mahout has implemented matrix factorization based on Stochastic gradient descent is a gradient descent optimization method for minimizing an objective function that is written as a su of differentiable functions. - Q(w) = sum(Q_i(w)), + Q(w) = sum(Q_i(w)), where w is the parameters to be estimated, Q(w) is the objective function that could be expressed as sum of differentiable functions, - Q_i(w) is associated with the i-th observation in the data set + Q_i(w) is associated with the i-th observation in the data set In practice, w is estimated using an iterative method at each single sample until an approximate miminum is obtained, @@ -67,9 +67,9 @@ where Q(u, i) is the objecive function for user u and item i, lambda is the regularization parameter to prevent overfitting. The algorithm is sketched as follows: - - init U and M with randomized value between 0.0 and 1.0 with standard Gaussian distribution - + + init U and M with randomized value between 0.0 and 1.0 with standard Gaussian distribution + for(iter = 0; iter < numIterations; iter++) { for(user u and item i with rating R[u,i]) @@ -80,11 +80,11 @@ The algorithm is sketched as follows: // p is the number of features for(f = 0; f < p; f++) { NU[u,f] = U[u,f] - alpha * d(Q(u,i))/d(U[u,f]) //optimize U[u,f] - = U[u, f] + alpha * (e(u,i) * M[i,f] - lambda * U[u,f]) + = U[u, f] + alpha * (e(u,i) * M[i,f] - lambda * U[u,f]) } for(f = 0; f < p; f++) { - M[i,f] = M[i,f] - alpha * d(Q(u,i))/d(M[i,f]) //optimize M[i,f] - = M[i,f] + alpha * (e(u,i) * U[u,f] - lambda * M[i,f]) + M[i,f] = M[i,f] - alpha * d(Q(u,i))/d(M[i,f]) //optimize M[i,f] + = M[i,f] + alpha * (e(u,i) * U[u,f] - lambda * M[i,f]) } U[u,] = NU[u,] } @@ -92,14 +92,14 @@ The algorithm is sketched as follows: ## SVD++ -SVD++ is an enhancement of the SGD matrix factorization. +SVD++ is an enhancement of the SGD matrix factorization. -It could be considered as an integration of latent factor model and neighborhood based model, considering not only how users rate, but also who has rated what. +It could be considered as an integration of latent factor model and neighborhood based model, considering not only how users rate, but also who has rated what. + +The complete model is a sum of 3 sub-models with complete prediction formula as follows: -The complete model is a sum of 3 sub-models with complete prediction formula as follows: - pr(u,i) = b[u,i] + fm + nm //user u and item i - + pr(u,i) is the predicted rating of user u on item i, b[u,i] = U + b(u) + b(i) fm = (q[i,]) * (p[u,] + pow(|N(u)|, -0.5) * sum(y[j,])), j is an item in N(u) @@ -109,7 +109,7 @@ The associated regularized squared error function to be minimized is: {sum((r[u,i] - pr[u,i]) * (r[u,i] - pr[u,i])) - lambda * (b(u) * b(u) + b(i) * b(i) + ||q[i,]||^2 + ||p[u,]||^2 + sum(||y[j,]||^2) + sum(w[i,j0] * w[i,j0]) + sum(c[i,j1] * c[i,j1]))} -b[u,i] is the baseline estimate of user u's predicted rating on item i. U is users' overall average rating and b(u) and b(i) indicate the observed deviations of user u and item i's ratings from average. +b[u,i] is the baseline estimate of user u's predicted rating on item i. U is users' overall average rating and b(u) and b(i) indicate the observed deviations of user u and item i's ratings from average. The baseline estimate is to adjust for the user and item effects - i.e, systematic tendencies for some users to give higher ratings than others and tendencies for some items to receive higher ratings than other items. @@ -117,29 +117,29 @@ for some items to receive higher ratings than other items. fm is the latent factor model to capture the interactions between user and item via a feature layer. q[i,] is the feature vector of item i, and the rest part of the formula represents user u with a user feature vector and a sum of features of items in N(u), N(u) is the set of items that user u have expressed preference, y[j,] is feature vector of an item in N(u). -nm is an extension of the classic item-based neighborhood model. -It captures not only the user's explicit ratings but also the user's implicit preferences. R(i;u;k) is the set of items that have got explicit rating from user u and only retain top k most similar items. r[u,j0] is the actual rating of user u on item j0, +nm is an extension of the classic item-based neighborhood model. +It captures not only the user's explicit ratings but also the user's implicit preferences. R(i;u;k) is the set of items that have got explicit rating from user u and only retain top k most similar items. r[u,j0] is the actual rating of user u on item j0, b[u,j0] is the corresponding baseline estimate. -The difference between r[u,j0] and b[u,j0] is weighted by a parameter w[i,j0], which could be thought as the similarity between item i and j0. +The difference between r[u,j0] and b[u,j0] is weighted by a parameter w[i,j0], which could be thought as the similarity between item i and j0. N[i;u;k] is the top k most similar items that have got the user's preference. -c[i;j1] is the paramter to be estimated. +c[i;j1] is the paramter to be estimated. -The value of w[i,j0] and c[i,j1] could be treated as the significance of the +The value of w[i,j0] and c[i,j1] could be treated as the significance of the user's explicit rating and implicit preference respectively. The parameters b, y, q, w, c are to be determined by minimizing the the associated regularized squared error function through gradient descent. We loop over all known ratings and for a given training case r[u,i], we apply gradient descent on the error function and modify the parameters by moving in the opposite direction of the gradient. For a complete analysis of the SVD++ algorithm, please refer to the paper [Yehuda Koren: Factorization Meets the Neighborhood: a Multifaceted Collaborative Filtering Model, KDD 2008](http://research.yahoo.com/files/kdd08koren.pdf). - -In Mahout,SVDPlusPlusFactorizer class is a simplified implementation of the SVD++ algorithm.It mainly uses the latent factor model with item feature vector, user feature vector and user's preference, with pr(u,i) = fm = (q[i,]) \* (p[u,] + pow(|N(u)|, -0.5) * sum(y[j,])) and the parameters to be determined are q, p, y. + +In Mahout,SVDPlusPlusFactorizer class is a simplified implementation of the SVD++ algorithm.It mainly uses the latent factor model with item feature vector, user feature vector and user's preference, with pr(u,i) = fm = (q[i,]) \* (p[u,] + pow(|N(u)|, -0.5) * sum(y[j,])) and the parameters to be determined are q, p, y. The update to q, p, y in each gradient descent step is: err(u,i) = r[u,i] - pr[u,i] - q[i,] = q[i,] + alpha * (err(u,i) * (p[u,] + pow(|N(u)|, -0.5) * sum(y[j,])) - lamda * q[i,]) + q[i,] = q[i,] + alpha * (err(u,i) * (p[u,] + pow(|N(u)|, -0.5) * sum(y[j,])) - lamda * q[i,]) p[u,] = p[u,] + alpha * (err(u,i) * q[i,] - lambda * p[u,]) for j that is an item in N(u): y[j,] = y[j,] + alpha * (err(u,i) * pow(|N(u)|, -0.5) * q[i,] - lambda * y[j,]) @@ -148,31 +148,31 @@ where alpha is the learning rate of gradient descent, N(u) is the items that use ## Parallel SGD -Mahout has a parallel SGD implementation in ParallelSGDFactorizer class. It shuffles the user ratings in every iteration and -generates splits on the shuffled ratings. Each split is handled by a thread to update the user features and item features using -vanilla SGD. +Mahout has a parallel SGD implementation in ParallelSGDFactorizer class. It shuffles the user ratings in every iteration and +generates splits on the shuffled ratings. Each split is handled by a thread to update the user features and item features using +vanilla SGD. -The implementation could be traced back to a lock-free version of SGD based on paper +The implementation could be traced back to a lock-free version of SGD based on paper [Hogwild!: A Lock-Free Approach to Parallelizing Stochastic Gradient Descent](http://www.eecs.berkeley.edu/~brecht/papers/hogwildTR.pdf). ## ALSWR -ALSWR is an iterative algorithm to solve the low rank factorization of user feature matrix U and item feature matrix M. +ALSWR is an iterative algorithm to solve the low rank factorization of user feature matrix U and item feature matrix M. The loss function to be minimized is formulated as the sum of squared errors plus [Tikhonov regularization](http://en.wikipedia.org/wiki/Tikhonov_regularization): L(R, U, M) = sum(pow((R[u,i] - U[u,]* (M[i,]^t)), 2)) + lambda * (sum(n(u) * ||U[u,]||^2) + sum(n(i) * ||M[i,]||^2)) - -At the beginning of the algorithm, M is initialized with the average item ratings as its first row and random numbers for the rest row. -In every iteration, we fix M and solve U by minimization of the cost function L(R, U, M), then we fix U and solve M by the minimization of +At the beginning of the algorithm, M is initialized with the average item ratings as its first row and random numbers for the rest row. + +In every iteration, we fix M and solve U by minimization of the cost function L(R, U, M), then we fix U and solve M by the minimization of the cost function similarly. The iteration stops until a certain stopping criteria is met. -To solve the matrix U when M is given, each user's feature vector is calculated by resolving a regularized linear least square error function +To solve the matrix U when M is given, each user's feature vector is calculated by resolving a regularized linear least square error function using the items the user has rated and their feature vectors: - 1/2 * d(L(R,U,M)) / d(U[u,f]) = 0 + 1/2 * d(L(R,U,M)) / d(U[u,f]) = 0 -Similary, when M is updated, we resolve a regularized linear least square error function using feature vectors of the users that have rated the +Similary, when M is updated, we resolve a regularized linear least square error function using feature vectors of the users that have rated the item and their feature vectors: 1/2 * d(L(R,U,M)) / d(M[i,f]) = 0 @@ -184,6 +184,5 @@ Mahout also offers a [parallel map-reduce implementation](https://mahout.apache. # Reference: [Stochastic gradient descent](http://en.wikipedia.org/wiki/Stochastic_gradient_descent) - -[ALSWR](http://www.hpl.hp.com/personal/Robert_Schreiber/papers/2008%20AAIM%20Netflix/netflix_aaim08%28submitted%29.pdf) +[ALSWR](http://www.hpl.hp.com/personal/Robert_Schreiber/papers/2008%20AAIM%20Netflix/netflix_aaim08%28submitted%29.pdf) diff --git a/website/Map_Reduce_Folder/recommender/recommender-documentation.md b/website/Map_Reduce_Folder/recommender/recommender-documentation.md index a29186714f..8997833b79 100644 --- a/website/Map_Reduce_Folder/recommender/recommender-documentation.md +++ b/website/Map_Reduce_Folder/recommender/recommender-documentation.md @@ -39,7 +39,7 @@ Top-level packages define the Mahout interfaces to these key abstractions: Subpackages of *org.apache.mahout.cf.taste.impl* hold implementations of these interfaces. These are the pieces from which you will build your own -recommendation engine. That's it! +recommendation engine. That's it! ## Architecture @@ -124,23 +124,23 @@ Now we create a **UserNeighborhood** algorithm. Here we use nearest-3: UserNeighborhood neighborhood = new NearestNUserNeighborhood(3, userSimilarity, model);{code} - + Now we can create our **Recommender**, and add a caching decorator: - + Recommender recommender = new GenericUserBasedRecommender(model, neighborhood, userSimilarity); Recommender cachingRecommender = new CachingRecommender(recommender); - + Now we can get 10 recommendations for user ID "1234" — done! List recommendations = cachingRecommender.recommend(1234, 10); - + ## Item-based Recommender - + We could have created an item-based recommender instead. Item-based recommenders base recommendation not on user similarity, but on item similarity. In theory these are about the same approach to the problem, @@ -149,19 +149,19 @@ relatively fixed, more so than the similarity of two users. So, item-based recommenders can use pre-computed similarity values in the computations, which make them much faster. For large data sets, item-based recommenders are more appropriate. - + Let's start over, again with a **FileDataModel** to start: - + DataModel model = new FileDataModel(new File("data.txt")); - + We'll also need an **ItemSimilarity**. We could use **PearsonCorrelationSimilarity**, which computes item similarity in realtime, but, this is generally too slow to be useful. Instead, in a real application, you would feed a list of pre-computed correlations to a -**GenericItemSimilarity**: - +**GenericItemSimilarity**: + // Construct the list of pre-computed correlations Collection correlations = @@ -170,9 +170,9 @@ application, you would feed a list of pre-computed correlations to a new GenericItemSimilarity(correlations); - + Then we can finish as before to produce recommendations: - + Recommender recommender = new GenericItemBasedRecommender(model, itemSimilarity); @@ -220,7 +220,7 @@ code and third-party code you use doesn't call this method. Also consider the following tips: * Use **CachingRecommender** on top of your custom **Recommender** implementation. -* When using **JDBCDataModel**, make sure you wrap it with the **ReloadFromJDBCDataModel** to load data into memory!. +* When using **JDBCDataModel**, make sure you wrap it with the **ReloadFromJDBCDataModel** to load data into memory!. ### Algorithm Performance: Which One Is Best? diff --git a/website/Map_Reduce_Folder/recommender/recommender-first-timer-faq.md b/website/Map_Reduce_Folder/recommender/recommender-first-timer-faq.md index 3e40f6d4c0..48aa32db0f 100644 --- a/website/Map_Reduce_Folder/recommender/recommender-first-timer-faq.md +++ b/website/Map_Reduce_Folder/recommender/recommender-first-timer-faq.md @@ -17,7 +17,7 @@ For the interested, these topics are treated in detail in the book [Mahout in Ac Don't start with a distributed, Hadoop-based recommender; take on that complexity only if necessary. Start with non-distributed recommenders. It -is simpler, has fewer requirements, and is more flexible. +is simpler, has fewer requirements, and is more flexible. As a crude rule of thumb, a system with up to 100M user-item associations (ratings, preferences) should "fit" onto one modern server machine with 4GB @@ -52,4 +52,4 @@ top a ReloadFromJDBCDataModel. This should give a reasonable starter system which responds fast. The nature of the system is that new data comes in from the file or database -only periodically -- perhaps on the order of minutes. +only periodically -- perhaps on the order of minutes. diff --git a/website/Map_Reduce_Folder/recommender/userbased-5-minutes.md b/website/Map_Reduce_Folder/recommender/userbased-5-minutes.md index 3536ddcbc4..54c9690158 100644 --- a/website/Map_Reduce_Folder/recommender/userbased-5-minutes.md +++ b/website/Map_Reduce_Folder/recommender/userbased-5-minutes.md @@ -17,7 +17,7 @@ Create a java project in your favorite IDE and make sure mahout is on the classp Mahout's recommenders expect interactions between users and items as input. The easiest way to supply such data to Mahout is in the form of a textfile, where every line has the format *userID,itemID,value*. Here *userID* and *itemID* refer to a particular user and a particular item, and *value* denotes the strength of the interaction (e.g. the rating given to a movie). -In this example, we'll use some made up data for simplicity. Create a file called "dataset.csv" and copy the following example interactions into the file. +In this example, we'll use some made up data for simplicity. Create a file called "dataset.csv" and copy the following example interactions into the file.
 1,10,1.0
@@ -79,9 +79,9 @@ Now we have all the pieces to create our recommender:
 
 UserBasedRecommender recommender = new GenericUserBasedRecommender(model, neighborhood, similarity);
 
- + We can easily ask the recommender for recommendations now. If we wanted to get three items recommended for the user with *userID* 2, we would do it like this: - +
 List recommendations = recommender.recommend(2, 3);
@@ -110,7 +110,7 @@ UserNeighborhood neighborhood = new ThresholdUserNeighborhood(0.1, similarity, d
 return new GenericUserBasedRecommender(dataModel, neighborhood, similarity);
 
-Now we have to create the code for the test. We'll check how much the recommender misses the real interaction strength on average. We employ an *AverageAbsoluteDifferenceRecommenderEvaluator* for this. The following code shows how to put the pieces together and run a hold-out test: +Now we have to create the code for the test. We'll check how much the recommender misses the real interaction strength on average. We employ an *AverageAbsoluteDifferenceRecommenderEvaluator* for this. The following code shows how to put the pieces together and run a hold-out test:
 DataModel model = new FileDataModel(new File("/path/to/dataset.csv"));
@@ -120,15 +120,4 @@ double result = evaluator.evaluate(builder, null, model, 0.9, 1.0);
 System.out.println(result);
 
-Note: if you run this test multiple times, you will get different results, because the splitting into trainingset and testset is done randomly. - - - - - - - - - - - +Note: if you run this test multiple times, you will get different results, because the splitting into trainingset and testset is done randomly. diff --git a/website/Map_Reduce_Folder/sparkbindings/faq.md b/website/Map_Reduce_Folder/sparkbindings/faq.md index 422847d483..caf1a224d6 100644 --- a/website/Map_Reduce_Folder/sparkbindings/faq.md +++ b/website/Map_Reduce_Folder/sparkbindings/faq.md @@ -10,44 +10,40 @@ redirect_from: **Q: Mahout Spark shell doesn't start; "ClassNotFound" problems or various classpath problems.** -**A:** So far as of the time of this writing all reported problems starting the Spark shell in Mahout were revolving -around classpath issues one way or another. +**A:** So far as of the time of this writing all reported problems starting the Spark shell in Mahout were revolving +around classpath issues one way or another. -If you are getting method signature like errors, most probably you have mismatch between Mahout's Spark dependency +If you are getting method signature like errors, most probably you have mismatch between Mahout's Spark dependency and actual Spark installed. (At the time of this writing the HEAD depends on Spark 1.1.0) but check mahout/pom.xml. -Troubleshooting general classpath issues is pretty straightforward. Since Mahout is using Spark's installation -and its classpath as reported by Spark itself for Spark-related dependencies, it is important to make sure +Troubleshooting general classpath issues is pretty straightforward. Since Mahout is using Spark's installation +and its classpath as reported by Spark itself for Spark-related dependencies, it is important to make sure the classpath is sane and is made available to Mahout: 1. Check Spark is of correct version (same as in Mahout's poms), is compiled and SPARK_HOME is set. 2. Check Mahout is compiled and MAHOUT_HOME is set. -3. Run `$SPARK_HOME/bin/compute-classpath.sh` and make sure it produces sane result with no errors. -If it outputs something other than a straightforward classpath string, most likely Spark is not compiled/set correctly (later spark versions require +3. Run `$SPARK_HOME/bin/compute-classpath.sh` and make sure it produces sane result with no errors. +If it outputs something other than a straightforward classpath string, most likely Spark is not compiled/set correctly (later spark versions require `sbt/sbt assembly` to be run, simply runnig `sbt/sbt publish-local` is not enough any longer). 4. Run `$MAHOUT_HOME/bin/mahout -spark classpath` and check that path reported in step (3) is included. -**Q: I am using the command line Mahout jobs that run on Spark or am writing my own application that uses -Mahout's Spark code. When I run the code on my cluster I get ClassNotFound or signature errors during serialization. +**Q: I am using the command line Mahout jobs that run on Spark or am writing my own application that uses +Mahout's Spark code. When I run the code on my cluster I get ClassNotFound or signature errors during serialization. What's wrong?** - -**A:** The Spark artifacts in the maven ecosystem may not match the exact binary you are running on your cluster. This may -cause class name or version mismatches. In this case you may wish + +**A:** The Spark artifacts in the maven ecosystem may not match the exact binary you are running on your cluster. This may +cause class name or version mismatches. In this case you may wish to build Spark yourself to guarantee that you are running exactly what you are building Mahout against. To do this follow these steps in order: -1. Build Spark with maven, but **do not** use the "package" target as described on the Spark site. Build with the "clean install" target instead. +1. Build Spark with maven, but **do not** use the "package" target as described on the Spark site. Build with the "clean install" target instead. Something like: "mvn clean install -Dhadoop1.2.1" or whatever your particular build options are. This will put the jars for Spark in the local maven cache. 2. Deploy **your** Spark build to your cluster and test it there. -3. Build Mahout. This will cause maven to pull the jars for Spark from the local maven cache and may resolve missing +3. Build Mahout. This will cause maven to pull the jars for Spark from the local maven cache and may resolve missing or mis-identified classes. 4. if you are building your own code do so against the local builds of Spark and Mahout. **Q: The implicit SparkContext 'sc' does not work in the Mahout spark-shell.** -**A:** In the Mahout spark-shell the SparkContext is called 'sdc', where the 'd' stands for distributed. - - - - +**A:** In the Mahout spark-shell the SparkContext is called 'sdc', where the 'd' stands for distributed. diff --git a/website/Map_Reduce_Folder/sparkbindings/home.md b/website/Map_Reduce_Folder/sparkbindings/home.md index 79f02a1110..80a8e8878e 100644 --- a/website/Map_Reduce_Folder/sparkbindings/home.md +++ b/website/Map_Reduce_Folder/sparkbindings/home.md @@ -12,7 +12,7 @@ redirect_from: ## What is Scala & Spark Bindings? In short, Scala & Spark Bindings for Mahout is Scala DSL and algebraic optimizer of something like this (actual formula from **(d)spca**) - + `\[\mathbf{G}=\mathbf{B}\mathbf{B}^{\top}-\mathbf{C}-\mathbf{C}^{\top}+\mathbf{s}_{q}\mathbf{s}_{q}^{\top}\boldsymbol{\xi}^{\top}\boldsymbol{\xi}\]` @@ -23,23 +23,23 @@ Mahout Scala & Spark Bindings expression of the above: val g = bt.t %*% bt - c - c.t + (s_q cross s_q) * (xi dot xi) -The main idea is that a scientist writing algebraic expressions cannot care less of distributed +The main idea is that a scientist writing algebraic expressions cannot care less of distributed operation plans and works **entirely on the logical level** just like he or she would do with R. -Another idea is decoupling logical expression from distributed back-end. As more back-ends are added, +Another idea is decoupling logical expression from distributed back-end. As more back-ends are added, this implies **"write once, run everywhere"**. The linear algebra side works with scalars, in-core vectors and matrices, and Mahout Distributed Row Matrices (DRMs). -The ecosystem of operators is built in the R's image, i.e. it follows R naming such as %*%, -colSums, nrow, length operating over vectors or matices. +The ecosystem of operators is built in the R's image, i.e. it follows R naming such as %*%, +colSums, nrow, length operating over vectors or matices. -Important part of Spark Bindings is expression optimizer. It looks at expression as a whole +Important part of Spark Bindings is expression optimizer. It looks at expression as a whole and figures out how it can be simplified, and which physical operators should be picked. For example, there are currently about 5 different physical operators performing DRM-DRM multiplication -picked based on matrix geometry, distributed dataset partitioning, orientation etc. -If we count in DRM by in-core combinations, that would be another 4, i.e. 9 total -- all of it for just +picked based on matrix geometry, distributed dataset partitioning, orientation etc. +If we count in DRM by in-core combinations, that would be another 4, i.e. 9 total -- all of it for just simple x %*% y logical notation. @@ -48,7 +48,7 @@ Please refer to the documentation for details. ## Status -This environment addresses mostly R-like Linear Algebra optmizations for +This environment addresses mostly R-like Linear Algebra optmizations for Spark, Flink and H20. @@ -61,7 +61,7 @@ Spark, Flink and H20. * In-core ([ssvd]) and Distributed ([dssvd]) Stochastic SVD -- guinea pigs -- see the bindings manual * In-core ([spca]) and Distributed ([dspca]) Stochastic PCA -- guinea pigs -- see the bindings manual -* Distributed thin QR decomposition ([dqrThin]) -- guinea pig -- see the bindings manual +* Distributed thin QR decomposition ([dqrThin]) -- guinea pig -- see the bindings manual * [Current list of algorithms](https://mahout.apache.org/users/basics/algorithms.html) [ssvd]: https://github.com/apache/mahout/blob/trunk/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/SSVD.scala @@ -71,7 +71,7 @@ Spark, Flink and H20. [dqrThin]: https://github.com/apache/mahout/blob/trunk/spark/src/main/scala/org/apache/mahout/sparkbindings/decompositions/DQR.scala -## Related history of note +## Related history of note * CLI and Driver for Spark version of item similarity -- [MAHOUT-1541](https://issues.apache.org/jira/browse/MAHOUT-1541) * Command line interface for generalizable Spark pipelines -- [MAHOUT-1569](https://issues.apache.org/jira/browse/MAHOUT-1569) @@ -83,20 +83,16 @@ Spark, Flink and H20. * Full abstraction of DRM apis and algorithms from a distributed engine -- [MAHOUT-1529](https://issues.apache.org/jira/browse/MAHOUT-1529) * Port Naive Bayes -- [MAHOUT-1493](https://issues.apache.org/jira/browse/MAHOUT-1493) -## Work in progress +## Work in progress * Text-delimited files for input and output -- [MAHOUT-1568](https://issues.apache.org/jira/browse/MAHOUT-1568) * *Your issue here!* - - - - - diff --git a/website/Map_Reduce_Folder/sparkbindings/play-with-shell.md b/website/Map_Reduce_Folder/sparkbindings/play-with-shell.md index 30a48fb4b9..408114e424 100644 --- a/website/Map_Reduce_Folder/sparkbindings/play-with-shell.md +++ b/website/Map_Reduce_Folder/sparkbindings/play-with-shell.md @@ -20,25 +20,25 @@ We'll use an excerpt of a publicly available [dataset about cereals](http://lib. Name | protein | fat | carbo | sugars | rating :-----------------------|:--------|:----|:------|:-------|:--------- Apple Cinnamon Cheerios | 2 | 2 | 10.5 | 10 | 29.509541 -Cap'n'Crunch | 1 | 2 | 12 | 12 | 18.042851 +Cap'n'Crunch | 1 | 2 | 12 | 12 | 18.042851 Cocoa Puffs | 1 | 1 | 12 | 13 | 22.736446 -Froot Loops | 2 | 1 | 11 | 13 | 32.207582 +Froot Loops | 2 | 1 | 11 | 13 | 32.207582 Honey Graham Ohs | 1 | 2 | 12 | 11 | 21.871292 -Wheaties Honey Gold | 2 | 1 | 16 | 8 | 36.187559 +Wheaties Honey Gold | 2 | 1 | 16 | 8 | 36.187559 Cheerios | 6 | 2 | 17 | 1 | 50.764999 Clusters | 3 | 2 | 13 | 7 | 40.400208 -Great Grains Pecan | 3 | 3 | 13 | 4 | 45.811716 +Great Grains Pecan | 3 | 3 | 13 | 4 | 45.811716 ## Installing Mahout & Spark on your local machine -We describe how to do a quick toy setup of Spark & Mahout on your local machine, so that you can run this example and play with the shell. +We describe how to do a quick toy setup of Spark & Mahout on your local machine, so that you can run this example and play with the shell. 1. Download [Apache Spark 1.6.2](http://d3kbcqa49mib13.cloudfront.net/spark-1.6.2-bin-hadoop2.6.tgz) and unpack the archive file 1. Change to the directory where you unpacked Spark and type ```sbt/sbt assembly``` to build it 1. Create a directory for Mahout somewhere on your machine, change to there and checkout the master branch of Apache Mahout from GitHub ```git clone https://github.com/apache/mahout mahout``` 1. Change to the ```mahout``` directory and build mahout using ```mvn -DskipTests clean install``` - + ## Starting Mahout's Spark shell 1. Goto the directory where you unpacked Spark and type ```sbin/start-all.sh``` to locally start Spark @@ -47,8 +47,8 @@ We describe how to do a quick toy setup of Spark & Mahout on your local machine, export SPARK_HOME=[directory where you unpacked Spark] export MASTER=[url of the Spark master]
- 1. Finally, change to the directory where you unpacked Mahout and type ```bin/mahout spark-shell```, -you should see the shell starting and get the prompt ```mahout> ```. Check + 1. Finally, change to the directory where you unpacked Mahout and type ```bin/mahout spark-shell```, +you should see the shell starting and get the prompt ```mahout> ```. Check [FAQ](http://mahout.apache.org/users/sparkbindings/faq.html) for further troubleshooting. ## Implementation @@ -73,15 +73,15 @@ val drmData = drmParallelize(dense( numPartitions = 2); -Have a look at this matrix. The first four columns represent the ingredients -(our features) and the last column (the rating) is the target variable for -our regression. [Linear regression](https://en.wikipedia.org/wiki/Linear_regression) -assumes that the **target variable** `\(\mathbf{y}\)` is generated by the -linear combination of **the feature matrix** `\(\mathbf{X}\)` with the +Have a look at this matrix. The first four columns represent the ingredients +(our features) and the last column (the rating) is the target variable for +our regression. [Linear regression](https://en.wikipedia.org/wiki/Linear_regression) +assumes that the **target variable** `\(\mathbf{y}\)` is generated by the +linear combination of **the feature matrix** `\(\mathbf{X}\)` with the **parameter vector** `\(\boldsymbol{\beta}\)` plus the - **noise** `\(\boldsymbol{\varepsilon}\)`, summarized in the formula -`\(\mathbf{y}=\mathbf{X}\boldsymbol{\beta}+\boldsymbol{\varepsilon}\)`. -Our goal is to find an estimate of the parameter vector + **noise** `\(\boldsymbol{\varepsilon}\)`, summarized in the formula +`\(\mathbf{y}=\mathbf{X}\boldsymbol{\beta}+\boldsymbol{\varepsilon}\)`. +Our goal is to find an estimate of the parameter vector `\(\boldsymbol{\beta}\)` that explains the data very well. As a first step, we extract `\(\mathbf{X}\)` and `\(\mathbf{y}\)` from our data matrix. We get *X* by slicing: we take all rows (denoted by ```::```) and the first four columns, which have the ingredients in milligrams as content. Note that the result is again a DRM. The shell will not execute this code yet, it saves the history of operations and defers the execution until we really access a result. **Mahout's DSL automatically optimizes and parallelizes all operations on DRMs and runs them on Apache Spark.** @@ -96,7 +96,7 @@ Next, we extract the target variable vector *y*, the fifth column of the data ma val y = drmData.collect(::, 4) -Now we are ready to think about a mathematical way to estimate the parameter vector *β*. A simple textbook approach is [ordinary least squares (OLS)](https://en.wikipedia.org/wiki/Ordinary_least_squares), which minimizes the sum of residual squares between the true target variable and the prediction of the target variable. In OLS, there is even a closed form expression for estimating `\(\boldsymbol{\beta}\)` as +Now we are ready to think about a mathematical way to estimate the parameter vector *β*. A simple textbook approach is [ordinary least squares (OLS)](https://en.wikipedia.org/wiki/Ordinary_least_squares), which minimizes the sum of residual squares between the true target variable and the prediction of the target variable. In OLS, there is even a closed form expression for estimating `\(\boldsymbol{\beta}\)` as `\(\left(\mathbf{X}^{\top}\mathbf{X}\right)^{-1}\mathbf{X}^{\top}\mathbf{y}\)`. The first thing which we compute for this is `\(\mathbf{X}^{\top}\mathbf{X}\)`. The code for doing this in Mahout's scala DSL maps directly to the mathematical formula. The operation ```.t()``` transposes a matrix and analogous to R ```%*%``` denotes matrix multiplication. @@ -110,12 +110,12 @@ The same is true for computing `\(\mathbf{X}^{\top}\mathbf{y}\)`. We can simply val drmXty = drmX.t %*% y -We're nearly done. The next step we take is to fetch `\(\mathbf{X}^{\top}\mathbf{X}\)` and -`\(\mathbf{X}^{\top}\mathbf{y}\)` into the memory of our driver machine (we are targeting -features matrices that are tall and skinny , -so we can assume that `\(\mathbf{X}^{\top}\mathbf{X}\)` is small enough -to fit in). Then, we provide them to an in-memory solver (Mahout provides -the an analog to R's ```solve()``` for that) which computes ```beta```, our +We're nearly done. The next step we take is to fetch `\(\mathbf{X}^{\top}\mathbf{X}\)` and +`\(\mathbf{X}^{\top}\mathbf{y}\)` into the memory of our driver machine (we are targeting +features matrices that are tall and skinny , +so we can assume that `\(\mathbf{X}^{\top}\mathbf{X}\)` is small enough +to fit in). Then, we provide them to an in-memory solver (Mahout provides +the an analog to R's ```solve()``` for that) which computes ```beta```, our OLS estimate of the parameter vector `\(\boldsymbol{\beta}\)`.
@@ -125,16 +125,16 @@ val Xty = drmXty.collect(::, 0)
 val beta = solve(XtX, Xty)
 
-That's it! We have a implemented a distributed linear regression algorithm -on Apache Spark. I hope you agree that we didn't have to worry a lot about -parallelization and distributed systems. The goal of Mahout's linear algebra -DSL is to abstract away the ugliness of programming a distributed system -as much as possible, while still retaining decent performance and +That's it! We have a implemented a distributed linear regression algorithm +on Apache Spark. I hope you agree that we didn't have to worry a lot about +parallelization and distributed systems. The goal of Mahout's linear algebra +DSL is to abstract away the ugliness of programming a distributed system +as much as possible, while still retaining decent performance and scalability. -We can now check how well our model fits its training data. -First, we multiply the feature matrix `\(\mathbf{X}\)` by our estimate of -`\(\boldsymbol{\beta}\)`. Then, we look at the difference (via L2-norm) of +We can now check how well our model fits its training data. +First, we multiply the feature matrix `\(\mathbf{X}\)` by our estimate of +`\(\boldsymbol{\beta}\)`. Then, we look at the difference (via L2-norm) of the target variable `\(\mathbf{y}\)` to the fitted target variable:
@@ -142,18 +142,18 @@ val yFitted = (drmX %*% beta).collect(::, 0)
 (y - yFitted).norm(2)
 
-We hope that we could show that Mahout's shell allows people to interactively and incrementally write algorithms. We have entered a lot of individual commands, one-by-one, until we got the desired results. We can now refactor a little by wrapping our statements into easy-to-use functions. The definition of functions follows standard scala syntax. +We hope that we could show that Mahout's shell allows people to interactively and incrementally write algorithms. We have entered a lot of individual commands, one-by-one, until we got the desired results. We can now refactor a little by wrapping our statements into easy-to-use functions. The definition of functions follows standard scala syntax. -We put all the commands for ordinary least squares into a function ```ols```. +We put all the commands for ordinary least squares into a function ```ols```.
-def ols(drmX: DrmLike[Int], y: Vector) = 
+def ols(drmX: DrmLike[Int], y: Vector) =
   solve(drmX.t %*% drmX, drmX.t %*% y)(::, 0)
 
 
Note that DSL declares implicit `collect` if coersion rules require an in-core argument. Hence, we can simply -skip explicit `collect`s. +skip explicit `collect`s. Next, we define a function ```goodnessOfFit``` that tells how well a model fits the target variable: @@ -164,11 +164,11 @@ def goodnessOfFit(drmX: DrmLike[Int], beta: Vector, y: Vector) = { } -So far we have left out an important aspect of a standard linear regression -model. Usually there is a constant bias term added to the model. Without -that, our model always crosses through the origin and we only learn the -right angle. An easy way to add such a bias term to our model is to add a -column of ones to the feature matrix `\(\mathbf{X}\)`. +So far we have left out an important aspect of a standard linear regression +model. Usually there is a constant bias term added to the model. Without +that, our model always crosses through the origin and we only learn the +right angle. An easy way to add such a bias term to our model is to add a +column of ones to the feature matrix `\(\mathbf{X}\)`. The corresponding weight in the parameter vector will then be the bias term. Here is how we add a bias column: diff --git a/website/Map_Reduce_Folder/tutorials/cco-last-fm/index.md b/website/Map_Reduce_Folder/tutorials/cco-last-fm/index.md index de1139b39f..5e5100ea9c 100644 --- a/website/Map_Reduce_Folder/tutorials/cco-last-fm/index.md +++ b/website/Map_Reduce_Folder/tutorials/cco-last-fm/index.md @@ -8,14 +8,14 @@ redirect_from: Most recommender examples utilize the MovieLense dataset, but that relies only on ratings (which makes the recommender being demonstrated look less trivial). Right next to the MovieLense dataset is the LastFM data set. The LastFM dataset has ratings by user, friends of the user, bands listened to by user, and tags by user. This is the kind of exciting data set we’d like to work with! -Start by downloading the LastFM dataset from +Start by downloading the LastFM dataset from http://files.grouplens.org/datasets/hetrec2011/hetrec2011-lastfm-2k.zip I’m going to assume you’ve unzipped them to /path/to/lastfm/* -We’re going to use a new trick for creating our IndexedDataSets, the `apply` function. `apply` takes an `RDD[(String, String)]` that is an RDD of tuples where both elements are strings. We load RDDs, and use Spark to manipulate the RDDs into this form. The files from LastFM are tab seperated- but it should be noted, that this could easily be done from log files, but would just take a touch more Spark-Fu. +We’re going to use a new trick for creating our IndexedDataSets, the `apply` function. `apply` takes an `RDD[(String, String)]` that is an RDD of tuples where both elements are strings. We load RDDs, and use Spark to manipulate the RDDs into this form. The files from LastFM are tab seperated- but it should be noted, that this could easily be done from log files, but would just take a touch more Spark-Fu. -The second important thing to note is that the first element in each tuple is going to be the rows in the resulting matrix, the second element will be the column, and at that position there will be a one. The BiDictionary will automatically be created from the strings. -For those following along at home- [the full Scala worksheet](cco-lastfm.scala) might be easier than copying and pasting +The second important thing to note is that the first element in each tuple is going to be the rows in the resulting matrix, the second element will be the column, and at that position there will be a one. The BiDictionary will automatically be created from the strings. +For those following along at home- [the full Scala worksheet](cco-lastfm.scala) might be easier than copying and pasting from this page. ~~~ @@ -31,14 +31,14 @@ val userArtistsRDD = sc.textFile("/path/to/lastfm/user_artists.dat") .map(line => line.split("\t")) .map(a => (a(0), a(1))) .filter(_._1 != "userID") - + val userArtistsIDS = IndexedDatasetSpark.apply(userArtistsRDD)(sc) val userFriendsRDD = sc.textFile("/path/to/lastfm/user_friends.dat") .map(line => line.split("\t")) .map(a => (a(0), a(1))) .filter(_._1 != "userID") - + val userFriendsIDS = IndexedDatasetSpark.apply(userFriendsRDD)(sc) ~~~ {: .language-scala} @@ -46,7 +46,7 @@ val userFriendsIDS = IndexedDatasetSpark.apply(userFriendsRDD)(sc) How much easier was that?! In each RDD creations we: Load our data using sc.textFile - + sc.textFile("/path/to/lastfm/user_taggedartists.dat") Split the data into an array based on tabs (\t) @@ -61,25 +61,25 @@ Remove the header (the only line that will have “userID” in that position) .filter(_._1 != "userID") -Then we easily create an IndexedDataSet using the `apply` method. +Then we easily create an IndexedDataSet using the `apply` method. val userTagsIDS = IndexedDatasetSpark.apply(userTagsRDD)(sc) -Note the `(sc)` at the end. You may or may not need that. `sc` is the SparkContext and should be passed as an implicit parameter, however the REPL environment (e.g. Mahout Shell or notebooks) has a hard time with the implicits, so I had to pass it explicitly. +Note the `(sc)` at the end. You may or may not need that. `sc` is the SparkContext and should be passed as an implicit parameter, however the REPL environment (e.g. Mahout Shell or notebooks) has a hard time with the implicits, so I had to pass it explicitly. Now we compute our co-occurrence matrices: ```scala import org.apache.mahout.math.cf.SimilarityAnalysis val artistReccosLlrDrmListByArtist = SimilarityAnalysis.cooccurrencesIDSs( - Array(userArtistsIDS, userTagsIDS, userFriendsIDS), + Array(userArtistsIDS, userTagsIDS, userFriendsIDS), maxInterestingItemsPerThing = 20, - maxNumInteractions = 500, + maxNumInteractions = 500, randomSeed = 1234) ``` Let’s see an example of how this would work- -First we have a small problem. If you look at our original input files, the userIDs, artistIDs, and tags were all integers. We loaded them as strings and if you look at the BiDictionaries associated with each IDS, you’ll see they map the original integers as strings to the integer indices of our matrix. Not super helpful. There are other files which contain mappings from LastFM ID to human readable band and tag names. I could have sorted this out in the begining but I chose to do it on the backside because it is a bit of clever Spark/Scala only needed to work around a quirk in this particular dataset. We have to reverse map a few things if we want to input ‘human readable’ attributes, which I did. If this doesn’t make sense, please don’t be discouraged- the important part was above, this is just some magic for working with this dataset in a pretty way. +First we have a small problem. If you look at our original input files, the userIDs, artistIDs, and tags were all integers. We loaded them as strings and if you look at the BiDictionaries associated with each IDS, you’ll see they map the original integers as strings to the integer indices of our matrix. Not super helpful. There are other files which contain mappings from LastFM ID to human readable band and tag names. I could have sorted this out in the begining but I chose to do it on the backside because it is a bit of clever Spark/Scala only needed to work around a quirk in this particular dataset. We have to reverse map a few things if we want to input ‘human readable’ attributes, which I did. If this doesn’t make sense, please don’t be discouraged- the important part was above, this is just some magic for working with this dataset in a pretty way. First I load, and create incore maps from the mapping files: @@ -116,7 +116,7 @@ val kilroyUserArtists = svec( (userArtistsIDS.columnIDs.get(artistMap("Beck")).g (userArtistsIDS.columnIDs.get(artistMap("Talking Heads")).get, 1) :: (userArtistsIDS.columnIDs.get(artistMap("Tom Waits")).get, 1) :: (userArtistsIDS.columnIDs.get(artistMap("Waylon Jennings")).get, 1) :: - (userArtistsIDS.columnIDs.get(artistMap("Wu-Tang Clan")).get, 1) :: Nil, + (userArtistsIDS.columnIDs.get(artistMap("Wu-Tang Clan")).get, 1) :: Nil, cardinality = userArtistsIDS.columnIDs.size ) @@ -130,14 +130,14 @@ val kilroyUserTags = svec( cardinality = userTagsIDS.columnIDs.size) ``` -So what we have then is me typing in a name to `artistMap` where the keys are human readable names of my favorite bands, which returns the value which is the LastFM ID, which in turn is the key in the BiDictionary map, and returns the matrix position. I’m making a sparse vector where I want the index at the value I just fetched (which in an awry way refers to the artist I specified) to have the value 1. +So what we have then is me typing in a name to `artistMap` where the keys are human readable names of my favorite bands, which returns the value which is the LastFM ID, which in turn is the key in the BiDictionary map, and returns the matrix position. I’m making a sparse vector where I want the index at the value I just fetched (which in an awry way refers to the artist I specified) to have the value 1. -Same idea for the tags. +Same idea for the tags. -I now have two history vectors. I didn’t make one for the users table, because I don’t have any friends on LastFM yet. That’s about to change though, because I’m about to have some friends recommended to me. +I now have two history vectors. I didn’t make one for the users table, because I don’t have any friends on LastFM yet. That’s about to change though, because I’m about to have some friends recommended to me. val kilroysRecs = (artistReccosLlrDrmListByArtist(0).matrix %*% kilroyUserArtists + artistReccosLlrDrmListByArtist(1).matrix %*% kilroyUserTags).collect -Finally let’s sort that vector out and get some user ids and strengths. +Finally let’s sort that vector out and get some user ids and strengths. ```scala import org.apache.mahout.math.scalabindings.MahoutCollections._ import collection._ @@ -148,8 +148,8 @@ println(kilroysRecs(::, 0).toMap.toList.sortWith(_._2 > _._2).take(5)) ``` -`kilroysRecs` is actually a one column matrix, so we take that, and the convert it into something we can sort. We then take the top 5 suggestions. Keep in mind, this will return the Mahout user ID, which you would also have to reverse map back to the lastFM userID. The lastFM userID is just another Integer, and not particularly exciting so I left that out. +`kilroysRecs` is actually a one column matrix, so we take that, and the convert it into something we can sort. We then take the top 5 suggestions. Keep in mind, this will return the Mahout user ID, which you would also have to reverse map back to the lastFM userID. The lastFM userID is just another Integer, and not particularly exciting so I left that out. -If you wanted to recommend artists like a normal recommendation engine- you would change the first position in all of the input matrices to be “artistID”. This is left as an exercise to the user. +If you wanted to recommend artists like a normal recommendation engine- you would change the first position in all of the input matrices to be “artistID”. This is left as an exercise to the user. [Full Scala Worksheet](cco-lastfm.scala) diff --git a/website/Map_Reduce_Folder/tutorials/eigenfaces/index.md b/website/Map_Reduce_Folder/tutorials/eigenfaces/index.md index 71c909287f..68a37af68b 100644 --- a/website/Map_Reduce_Folder/tutorials/eigenfaces/index.md +++ b/website/Map_Reduce_Folder/tutorials/eigenfaces/index.md @@ -17,7 +17,7 @@ Deep learning will admittedly do better a recognizing and correctly classifying The advantage/use-case for the eigenfaces approach is when new faces are being regularly added. Even when building a production grade eigenfaces based system- neural networks still have a place- _idenitifying faces_ in images, and creating _centered and scaled_ images around -the face. This is scalable because we only need to train our neural network to detect, center, and scale faces once. E.g. +the face. This is scalable because we only need to train our neural network to detect, center, and scale faces once. E.g. a neural network would be deployed as a microservice, and then eigenfaces would be deployed as a microservice. A production version ends up looking something like this: @@ -26,7 +26,7 @@ A production version ends up looking something like this: - For each face:
a. Decompose face into linear combination of eigenfaces
b. Determine if linear combination vector is close enough to any exististing vector to declare a match
- c. If no match "add new person" to face corpus. + c. If no match "add new person" to face corpus. ### Get the data @@ -44,7 +44,7 @@ The first thing we're going to do is collect a set of 13,232 face images (250x25 --packages com.sksamuel.scrimage:scrimage-core_2.10:2.1.0, \ com.sksamuel.scrimage:scrimage-io-extra_2.10:2.1.0, \ com.sksamuel.scrimage:scrimage-filters_2.10:2.1.0 - + ### Create a DRM of Vectorized Images @@ -95,7 +95,7 @@ import java.io.File import javax.imageio.ImageIO val sampleImagePath = "/home/guest/lfw-deepfunneled/Aaron_Eckhart/Aaron_Eckhart_0001.jpg" -val sampleImage = ImageIO.read(new File(sampleImagePath)) +val sampleImage = ImageIO.read(new File(sampleImagePath)) val w = sampleImage.getWidth val h = sampleImage.getHeight @@ -119,7 +119,7 @@ If using Zeppelin, the following can be used to generate a fun table of the Eige ```python %python - + r = 4 c = 5 print '%html\n' + "".join(["" + "".join([ '' % (i + j) for j in range(0, c) ]) + "" for i in range(0, r * c, r +1 ) ]) + '
' diff --git a/website/Map_Reduce_Folder/tutorials/intro-cooccurrence-spark/index.md b/website/Map_Reduce_Folder/tutorials/intro-cooccurrence-spark/index.md index 9d7bf91601..d8257c5d92 100644 --- a/website/Map_Reduce_Folder/tutorials/intro-cooccurrence-spark/index.md +++ b/website/Map_Reduce_Folder/tutorials/intro-cooccurrence-spark/index.md @@ -8,16 +8,16 @@ redirect_from: # Intro to Cooccurrence Recommenders with Spark -Mahout provides several important building blocks for creating recommendations using Spark. *spark-itemsimilarity* can -be used to create "other people also liked these things" type recommendations and paired with a search engine can -personalize recommendations for individual users. *spark-rowsimilarity* can provide non-personalized content based +Mahout provides several important building blocks for creating recommendations using Spark. *spark-itemsimilarity* can +be used to create "other people also liked these things" type recommendations and paired with a search engine can +personalize recommendations for individual users. *spark-rowsimilarity* can provide non-personalized content based recommendations and when paired with a search engine can be used to personalize content based recommendations. ![image](http://s6.postimg.org/r0m8bpjw1/recommender_architecture.png) This is a simplified Lambda architecture with Mahout's *spark-itemsimilarity* playing the batch model building role and a search engine playing the realtime serving role. -You will create two collections, one for user history and one for item "indicators". Indicators are user interactions that lead to the wished for interaction. So for example if you wish a user to purchase something and you collect all users purchase interactions *spark-itemsimilarity* will create a purchase indicator from them. But you can also use other user interactions in a cross-cooccurrence calculation, to create purchase indicators. +You will create two collections, one for user history and one for item "indicators". Indicators are user interactions that lead to the wished for interaction. So for example if you wish a user to purchase something and you collect all users purchase interactions *spark-itemsimilarity* will create a purchase indicator from them. But you can also use other user interactions in a cross-cooccurrence calculation, to create purchase indicators. User history is used as a query on the item collection with its cooccurrence and cross-cooccurrence indicators (there may be several indicators). The primary interaction or action is picked to be the thing you want to recommend, other actions are believed to be corelated but may not indicate exactly the same user intent. For instance in an ecom recommender a purchase is a very good primary action, but you may also know product detail-views, or additions-to-wishlists. These can be considered secondary actions which may all be used to calculate cross-cooccurrence indicators. The user history that forms the recommendations query will contain recorded primary and secondary actions all targetted towards the correct indicator fields. @@ -34,20 +34,20 @@ Below are the command line jobs but the drivers and associated code can also be ## 1. spark-itemsimilarity *spark-itemsimilarity* is the Spark counterpart of the of the Mahout mapreduce job called *itemsimilarity*. It takes in elements of interactions, which have userID, itemID, and optionally a value. It will produce one of more indicator matrices created by comparing every user's interactions with every other user. The indicator matrix is an item x item matrix where the values are log-likelihood ratio strengths. For the legacy mapreduce version, there were several possible similarity measures but these are being deprecated in favor of LLR because in practice it performs the best. -Mahout's mapreduce version of itemsimilarity takes a text file that is expected to have user and item IDs that conform to +Mahout's mapreduce version of itemsimilarity takes a text file that is expected to have user and item IDs that conform to Mahout's ID requirements--they are non-negative integers that can be viewed as row and column numbers in a matrix. -*spark-itemsimilarity* also extends the notion of cooccurrence to cross-cooccurrence, in other words the Spark version will -account for multi-modal interactions and create cross-cooccurrence indicator matrices allowing the use of much more data in -creating recommendations or similar item lists. People try to do this by mixing different actions and giving them weights. +*spark-itemsimilarity* also extends the notion of cooccurrence to cross-cooccurrence, in other words the Spark version will +account for multi-modal interactions and create cross-cooccurrence indicator matrices allowing the use of much more data in +creating recommendations or similar item lists. People try to do this by mixing different actions and giving them weights. For instance they might say an item-view is 0.2 of an item purchase. In practice this is often not helpful. Spark-itemsimilarity's cross-cooccurrence is a more principled way to handle this case. In effect it scrubs secondary actions with the action you want -to recommend. +to recommend. spark-itemsimilarity Mahout 1.0 Usage: spark-itemsimilarity [options] - + Disconnected from the target VM, address: '127.0.0.1:64676', transport: 'socket' Input, output options -i | --input @@ -56,15 +56,15 @@ to recommend. Secondary input path for cross-similarity calculation, same restrictions as "--input" (optional). Default: empty. -o | --output Path for output, any local or HDFS supported URI (required) - + Algorithm control options: -mppu | --maxPrefs Max number of preferences to consider per user (optional). Default: 500 -m | --maxSimilaritiesPerItem Limit the number of similarities per item to this number (optional). Default: 100 - + Note: Only the Log Likelihood Ratio (LLR) is supported as a similarity measure. - + Input text file schema options: -id | --inDelim Input delimiter character (optional). Default: "[,\t]" @@ -78,15 +78,15 @@ to recommend. Column number (0 based Int) containing the item ID string (optional). Default: 1 -fc | --filterColumn Column number (0 based Int) containing the filter string (optional). Default: -1 for no filter - + Using all defaults the input is expected of the form: "userIDitemId" or "userIDitemIDany-text..." and all rows will be used - + File discovery options: -r | --recursive Searched the -i path recursively for files that match --filenamePattern (optional), Default: false -fp | --filenamePattern Regex to match in determining input files (optional). Default: filename in the --input option or "^part-.*" if --input is a directory - + Output text file schema options: -rd | --rowKeyDelim Separates the rowID key from the vector values list (optional). Default: "\t" @@ -97,22 +97,22 @@ to recommend. -os | --omitStrength Do not write the strength to the output files (optional), Default: false. This option is used to output indexable data for creating a search engine recommender. - + Default delimiters will produce output of the form: "itemID1itemID2:value2itemID10:value10..." - + Spark config options: -ma | --master Spark Master URL (optional). Default: "local". Note that you can specify the number of cores to get a performance improvement, for example "local[4]" -sem | --sparkExecutorMem Max Java heap available as "executor memory" on each node (optional). Default: 4g -rs | --randomSeed - + -h | --help prints this usage text This looks daunting but defaults to simple fairly sane values to take exactly the same input as legacy code and is pretty flexible. It allows the user to point to a single text file, a directory full of files, or a tree of directories to be traversed recursively. The files included can be specified with either a regex-style pattern or filename. The schema for the file is defined by column numbers, which map to the important bits of data including IDs and values. The files can even contain filters, which allow unneeded rows to be discarded or used for cross-cooccurrence calculations. -See ItemSimilarityDriver.scala in Mahout's spark module if you want to customize the code. +See ItemSimilarityDriver.scala in Mahout's spark module if you want to customize the code. ### Defaults in the _**spark-itemsimilarity**_ CLI @@ -134,16 +134,16 @@ This will use the "local" Spark context and will output the standard text versio ### How To Use Multiple User Actions -Often we record various actions the user takes for later analytics. These can now be used to make recommendations. -The idea of a recommender is to recommend the action you want the user to make. For an ecom app this might be -a purchase action. It is usually not a good idea to just treat other actions the same as the action you want to recommend. -For instance a view of an item does not indicate the same intent as a purchase and if you just mixed the two together you +Often we record various actions the user takes for later analytics. These can now be used to make recommendations. +The idea of a recommender is to recommend the action you want the user to make. For an ecom app this might be +a purchase action. It is usually not a good idea to just treat other actions the same as the action you want to recommend. +For instance a view of an item does not indicate the same intent as a purchase and if you just mixed the two together you might even make worse recommendations. It is tempting though since there are so many more views than purchases. With *spark-itemsimilarity* we can now use both actions. Mahout will use cross-action cooccurrence analysis to limit the views to ones that do predict purchases. -We do this by treating the primary action (purchase) as data for the indicator matrix and use the secondary action (view) -to calculate the cross-cooccurrence indicator matrix. +We do this by treating the primary action (purchase) as data for the indicator matrix and use the secondary action (view) +to calculate the cross-cooccurrence indicator matrix. -*spark-itemsimilarity* can read separate actions from separate files or from a mixed action log by filtering certain lines. For a mixed +*spark-itemsimilarity* can read separate actions from separate files or from a mixed action log by filtering certain lines. For a mixed action log of the form: u1,purchase,iphone @@ -186,7 +186,7 @@ Use the following options: ### Output -The output of the job will be the standard text version of two Mahout DRMs. This is a case where we are calculating +The output of the job will be the standard text version of two Mahout DRMs. This is a case where we are calculating cross-cooccurrence so a primary indicator matrix and cross-cooccurrence indicator matrix will be created out-path @@ -209,11 +209,11 @@ The cross-similarity-matrix will contain: galaxy\tnexus:1.7260924347106847 iphone:1.7260924347106847 ipad:1.7260924347106847 galaxy:1.7260924347106847 surface\tsurface:4.498681156950466 nexus:0.6795961471815897 -**Note:** You can run this multiple times to use more than two actions or you can use the underlying +**Note:** You can run this multiple times to use more than two actions or you can use the underlying SimilarityAnalysis.cooccurrence API, which will more efficiently calculate any number of cross-cooccurrence indicators. ### Log File Input - + A common method of storing data is in log files. If they are written using some delimiter they can be consumed directly by spark-itemsimilarity. For instance input of the form: 2014-06-23 14:46:53.115\tu1\tpurchase\trandom text\tiphone @@ -235,7 +235,7 @@ A common method of storing data is in log files. If they are written using some 2014-06-23 14:46:53.115\tu3\tview\trandom text\tnexus 2014-06-23 14:46:53.115\tu4\tview\trandom text\tiphone 2014-06-23 14:46:53.115\tu4\tview\trandom text\tipad - 2014-06-23 14:46:53.115\tu4\tview\trandom text\tgalaxy + 2014-06-23 14:46:53.115\tu4\tview\trandom text\tgalaxy Can be parsed with the following CLI and run on the cluster producing the same output as the above example. @@ -252,35 +252,35 @@ Can be parsed with the following CLI and run on the cluster producing the same o ## 2. spark-rowsimilarity -*spark-rowsimilarity* is the companion to *spark-itemsimilarity* the primary difference is that it takes a text file version of +*spark-rowsimilarity* is the companion to *spark-itemsimilarity* the primary difference is that it takes a text file version of a matrix of sparse vectors with optional application specific IDs and it finds similar rows rather than items (columns). Its use is -not limited to collaborative filtering. The input is in text-delimited form where there are three delimiters used. By +not limited to collaborative filtering. The input is in text-delimited form where there are three delimiters used. By default it reads (rowID<tab>columnID1:strength1<space>columnID2:strength2...) Since this job only supports LLR similarity, - which does not use the input strengths, they may be omitted in the input. It writes -(rowID<tab>rowID1:strength1<space>rowID2:strength2...) -The output is sorted by strength descending. The output can be interpreted as a row ID from the primary input followed + which does not use the input strengths, they may be omitted in the input. It writes +(rowID<tab>rowID1:strength1<space>rowID2:strength2...) +The output is sorted by strength descending. The output can be interpreted as a row ID from the primary input followed by a list of the most similar rows. The command line interface is: spark-rowsimilarity Mahout 1.0 Usage: spark-rowsimilarity [options] - + Input, output options -i | --input Input path, may be a filename, directory name, or comma delimited list of HDFS supported URIs (required) -o | --output Path for output, any local or HDFS supported URI (required) - + Algorithm control options: -mo | --maxObservations Max number of observations to consider per row (optional). Default: 500 -m | --maxSimilaritiesPerRow Limit the number of similarities per item to this number (optional). Default: 100 - + Note: Only the Log Likelihood Ratio (LLR) is supported as a similarity measure. Disconnected from the target VM, address: '127.0.0.1:49162', transport: 'socket' - + Output text file schema options: -rd | --rowKeyDelim Separates the rowID key from the vector values list (optional). Default: "\t" @@ -291,40 +291,40 @@ The command line interface is: -os | --omitStrength Do not write the strength to the output files (optional), Default: false. This option is used to output indexable data for creating a search engine recommender. - + Default delimiters will produce output of the form: "itemID1itemID2:value2itemID10:value10..." - + File discovery options: -r | --recursive Searched the -i path recursively for files that match --filenamePattern (optional), Default: false -fp | --filenamePattern Regex to match in determining input files (optional). Default: filename in the --input option or "^part-.*" if --input is a directory - + Spark config options: -ma | --master Spark Master URL (optional). Default: "local". Note that you can specify the number of cores to get a performance improvement, for example "local[4]" -sem | --sparkExecutorMem Max Java heap available as "executor memory" on each node (optional). Default: 4g -rs | --randomSeed - + -h | --help prints this usage text -See RowSimilarityDriver.scala in Mahout's spark module if you want to customize the code. +See RowSimilarityDriver.scala in Mahout's spark module if you want to customize the code. # 3. Using *spark-rowsimilarity* with Text Data -Another use case for *spark-rowsimilarity* is in finding similar textual content. For instance given the tags associated with +Another use case for *spark-rowsimilarity* is in finding similar textual content. For instance given the tags associated with a blog post, - which other posts have similar tags. In this case the columns are tags and the rows are posts. Since LLR is -the only similarity method supported this is not the optimal way to determine general "bag-of-words" document similarity. -LLR is used more as a quality filter than as a similarity measure. However *spark-rowsimilarity* will produce + which other posts have similar tags. In this case the columns are tags and the rows are posts. Since LLR is +the only similarity method supported this is not the optimal way to determine general "bag-of-words" document similarity. +LLR is used more as a quality filter than as a similarity measure. However *spark-rowsimilarity* will produce lists of similar docs for every doc if input is docs with lists of terms. The Apache [Lucene](http://lucene.apache.org) project provides several methods of [analyzing and tokenizing](http://lucene.apache.org/core/4_9_0/core/org/apache/lucene/analysis/package-summary.html#package_description) documents. # 4. Creating a Multimodal Recommender Using the output of *spark-itemsimilarity* and *spark-rowsimilarity* you can build a miltimodal cooccurrence and content based - recommender that can be used in both or either mode depending on indicators available and the history available at + recommender that can be used in both or either mode depending on indicators available and the history available at runtime for a user. Some slide describing this method can be found [here](http://occamsmachete.com/ml/2014/10/07/creating-a-unified-recommender-with-mahout-and-a-search-engine/) ## Requirements @@ -342,44 +342,44 @@ Indicators come in 3 types 2. **Content**: calculated from item metadata or content using *spark-rowsimilarity* 3. **Intrinsic**: assigned to items as metadata. Can be anything that describes the item. -The query for recommendations will be a mix of values meant to match one of your indicators. The query can be constructed -from user history and values derived from context (category being viewed for instance) or special precalculated data -(popularity rank for instance). This blending of indicators allows for creating many flavors or recommendations to fit +The query for recommendations will be a mix of values meant to match one of your indicators. The query can be constructed +from user history and values derived from context (category being viewed for instance) or special precalculated data +(popularity rank for instance). This blending of indicators allows for creating many flavors or recommendations to fit a very wide variety of circumstances. -With the right mix of indicators developers can construct a single query that works for completely new items and new users -while working well for items with lots of interactions and users with many recorded actions. In other words by adding in content and intrinsic +With the right mix of indicators developers can construct a single query that works for completely new items and new users +while working well for items with lots of interactions and users with many recorded actions. In other words by adding in content and intrinsic indicators developers can create a solution for the "cold-start" problem that gracefully improves with more user history -and as items have more interactions. It is also possible to create a completely content-based recommender that personalizes +and as items have more interactions. It is also possible to create a completely content-based recommender that personalizes recommendations. ## Example with 3 Indicators -You will need to decide how you store user action data so they can be processed by the item and row similarity jobs and -this is most easily done by using text files as described above. The data that is processed by these jobs is considered the -training data. You will need some amount of user history in your recs query. It is typical to use the most recent user history -but need not be exactly what is in the training set, which may include a greater volume of historical data. Keeping the user -history for query purposes could be done with a database by storing it in a users table. In the example above the two -collaborative filtering actions are "purchase" and "view", but let's also add tags (taken from catalog categories or other -descriptive metadata). +You will need to decide how you store user action data so they can be processed by the item and row similarity jobs and +this is most easily done by using text files as described above. The data that is processed by these jobs is considered the +training data. You will need some amount of user history in your recs query. It is typical to use the most recent user history +but need not be exactly what is in the training set, which may include a greater volume of historical data. Keeping the user +history for query purposes could be done with a database by storing it in a users table. In the example above the two +collaborative filtering actions are "purchase" and "view", but let's also add tags (taken from catalog categories or other +descriptive metadata). -We will need to create 1 cooccurrence indicator from the primary action (purchase) 1 cross-action cooccurrence indicator -from the secondary action (view) +We will need to create 1 cooccurrence indicator from the primary action (purchase) 1 cross-action cooccurrence indicator +from the secondary action (view) and 1 content indicator (tags). We'll have to run *spark-itemsimilarity* once and *spark-rowsimilarity* once. -We have described how to create the collaborative filtering indicators for purchase and view (the [How to use Multiple User -Actions](#multiple-actions) section) but tags will be a slightly different process. We want to use the fact that -certain items have tags similar to the ones associated with a user's purchases. This is not a collaborative filtering indicator -but rather a "content" or "metadata" type indicator since you are not using other users' history, only the -individual that you are making recs for. This means that this method will make recommendations for items that have +We have described how to create the collaborative filtering indicators for purchase and view (the [How to use Multiple User +Actions](#multiple-actions) section) but tags will be a slightly different process. We want to use the fact that +certain items have tags similar to the ones associated with a user's purchases. This is not a collaborative filtering indicator +but rather a "content" or "metadata" type indicator since you are not using other users' history, only the +individual that you are making recs for. This means that this method will make recommendations for items that have no collaborative filtering data, as happens with new items in a catalog. New items may have tags assigned but no one has purchased or viewed them yet. In the final query we will mix all 3 indicators. ##Content Indicator -To create a content-indicator we'll make use of the fact that the user has purchased items with certain tags. We want to find -items with the most similar tags. Notice that other users' behavior is not considered--only other item's tags. This defines a -content or metadata indicator. They are used when you want to find items that are similar to other items by using their +To create a content-indicator we'll make use of the fact that the user has purchased items with certain tags. We want to find +items with the most similar tags. Notice that other users' behavior is not considered--only other item's tags. This defines a +content or metadata indicator. They are used when you want to find items that are similar to other items by using their content or metadata, not by which users interacted with them. **Note**: It may be advisable to treat tags as cross-cooccurrence indicators but for the sake of an example they are treated here as content only. @@ -395,9 +395,9 @@ The full collection will look like the tags column from a catalog DB. For our ec 9446577dwomen tops chambray clothing casual ... -We'll use *spark-rowimilairity* because we are looking for similar rows, which encode items in this case. As with the -collaborative filtering indicators we use the --omitStrength option. The strengths created are -probabilistic log-likelihood ratios and so are used to filter unimportant similarities. Once the filtering or downsampling +We'll use *spark-rowimilairity* because we are looking for similar rows, which encode items in this case. As with the +collaborative filtering indicators we use the --omitStrength option. The strengths created are +probabilistic log-likelihood ratios and so are used to filter unimportant similarities. Once the filtering or downsampling is finished we no longer need the strengths. We will get an indicator matrix of the form: itemIDlist-of-item IDs @@ -407,30 +407,30 @@ This is a content indicator since it has found other items with similar content 3459860b3459860b 3459860b 6749860c 5959860a 3434860a 3477860a 9446577d9446577d 9496577d 0943577d 8346577d 9442277d 9446577e - ... - + ... + We now have three indicators, two collaborative filtering type and one content type. ## Multimodal Recommender Query -The actual form of the query for recommendations will vary depending on your search engine but the intent is the same. For a given user, map their history of an action or content to the correct indicator field and perform an OR'd query. +The actual form of the query for recommendations will vary depending on your search engine but the intent is the same. For a given user, map their history of an action or content to the correct indicator field and perform an OR'd query. -We have 3 indicators, these are indexed by the search engine into 3 fields, we'll call them "purchase", "view", and "tags". +We have 3 indicators, these are indexed by the search engine into 3 fields, we'll call them "purchase", "view", and "tags". We take the user's history that corresponds to each indicator and create a query of the form: Query: field: purchase; q:user's-purchase-history field: view; q:user's view-history field: tags; q:user's-tags-associated-with-purchases - -The query will result in an ordered list of items recommended for purchase but skewed towards items with similar tags to -the ones the user has already purchased. -This is only an example and not necessarily the optimal way to create recs. It illustrates how business decisions can be -translated into recommendations. This technique can be used to skew recommendations towards intrinsic indicators also. -For instance you may want to put personalized popular item recs in a special place in the UI. Create a popularity indicator +The query will result in an ordered list of items recommended for purchase but skewed towards items with similar tags to +the ones the user has already purchased. + +This is only an example and not necessarily the optimal way to create recs. It illustrates how business decisions can be +translated into recommendations. This technique can be used to skew recommendations towards intrinsic indicators also. +For instance you may want to put personalized popular item recs in a special place in the UI. Create a popularity indicator by tagging items with some category of popularity (hot, warm, cold for instance) then -index that as a new indicator field and include the corresponding value in a query +index that as a new indicator field and include the corresponding value in a query on the popularity field. If we use the ecom example but use the query to get "hot" recommendations it might look like this: Query: diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/classification/bankmarketing-example.md b/website/Map_Reduce_Folder/tutorials/map-reduce/classification/bankmarketing-example.md index ac3eabe873..8dfd550188 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/classification/bankmarketing-example.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/classification/bankmarketing-example.md @@ -1,6 +1,6 @@ --- layout: deprecated-docs -title: (Deprecated) +title: (Deprecated) redirect_from: - /documentation/tutorials/map-reduce/classification/bankmarketing-example - /documentation/tutorials/map-reduce/classification/bankmarketing-example.html @@ -35,11 +35,11 @@ of information such as age, job, marital status as well as information about the ### Code & Data -The bank marketing example code lives under +The bank marketing example code lives under *mahout-examples/src/main/java/org.apache.mahout.classifier.sgd.bankmarketing* -The data can be found at +The data can be found at *mahout-examples/src/main/resources/bank-full.csv* diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/classification/breiman-example.md b/website/Map_Reduce_Folder/tutorials/map-reduce/classification/breiman-example.md index 94ffb8ee38..2b7d470c67 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/classification/breiman-example.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/classification/breiman-example.md @@ -16,7 +16,7 @@ This page describes how to run the Breiman example, which implements the test pr * repeat *I* iterations * in each iteration do - * keep 10% of the dataset apart as a testing set + * keep 10% of the dataset apart as a testing set * build two forests using the training set, one with *m = int(log2(M) + 1)* (called Random-Input) and one with *m = 1* (called Single-Input) * choose the forest that gave the lowest oob error estimation to compute the test set error @@ -45,7 +45,7 @@ Finally, we build and evaluate our random forest classifier as follows: bin/mahout org.apache.mahout.classifier.df.BreimanExample -d /path/to/glass.data -ds /path/to/glass.info -i 10 -t 100 which builds 100 trees (-t argument) and repeats the test 10 iterations (-i -argument) +argument) The example outputs the following results: @@ -65,6 +65,3 @@ We can repeat this for a [Sonar](http://archive.ics.uci.edu/ml/datasets/Connecti The argument "60 N L" means 60 numerical(N) attributes, followed by the label (L). Analogous to the previous case, we run the evaluation as follows: bin/mahout org.apache.mahout.classifier.df.BreimanExample -d /path/to/sonar.all-data -ds /path/to/sonar.info -i 10 -t 100 - - - diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/classification/twenty-newsgroups.md b/website/Map_Reduce_Folder/tutorials/map-reduce/classification/twenty-newsgroups.md index 9db731902c..69402787c9 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/classification/twenty-newsgroups.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/classification/twenty-newsgroups.md @@ -27,7 +27,7 @@ the 20 newsgroups. * Mahout has been downloaded ([instructions here](https://mahout.apache.org/download/downloads.html)) * Maven is available * Your environment has the following variables: - * **HADOOP_HOME** Environment variables refers to where Hadoop lives + * **HADOOP_HOME** Environment variables refers to where Hadoop lives * **MAHOUT_HOME** Environment variables refers to where Mahout lives @@ -37,7 +37,7 @@ the 20 newsgroups. $ cd $HADOOP_HOME/bin $ ./start-all.sh - + Otherwise: $ export MAHOUT_LOCAL=true @@ -51,8 +51,8 @@ the 20 newsgroups. $ ./examples/bin/classify-20newsgroups.sh -4. You will be prompted to select a classification method algorithm: - +4. You will be prompted to select a classification method algorithm: + 1. Complement Naive Bayes 2. Naive Bayes 3. Stochastic Gradient Descent @@ -61,9 +61,9 @@ Select 1 and the the script will perform the following: 1. Create a working directory for the dataset and all input/output. 2. Download and extract the *20news-bydate.tar.gz* from the [20 newsgroups dataset](http://people.csail.mit.edu/jrennie/20Newsgroups/20news-bydate.tar.gz) to the working directory. -3. Convert the full 20 newsgroups dataset into a < Text, Text > SequenceFile. +3. Convert the full 20 newsgroups dataset into a < Text, Text > SequenceFile. 4. Convert and preprocesses the dataset into a < Text, VectorWritable > SequenceFile containing term frequencies for each document. -5. Split the preprocessed dataset into training and testing sets. +5. Split the preprocessed dataset into training and testing sets. 6. Train the classifier. 7. Test the classifier. @@ -109,71 +109,69 @@ Output should look something like: ## End to end commands to build a CBayes model for 20 newsgroups -The [20 newsgroups example script](https://github.com/apache/mahout/blob/master/examples/bin/classify-20newsgroups.sh) issues the following commands as outlined above. We can build a CBayes classifier from the command line by following the process in the script: +The [20 newsgroups example script](https://github.com/apache/mahout/blob/master/examples/bin/classify-20newsgroups.sh) issues the following commands as outlined above. We can build a CBayes classifier from the command line by following the process in the script: *Be sure that **MAHOUT_HOME**/bin and **HADOOP_HOME**/bin are in your **$PATH*** 1. Create a working directory for the dataset and all input/output. - + $ export WORK_DIR=/tmp/mahout-work-${USER} $ mkdir -p ${WORK_DIR} 2. Download and extract the *20news-bydate.tar.gz* from the [20newsgroups dataset](http://people.csail.mit.edu/jrennie/20Newsgroups/20news-bydate.tar.gz) to the working directory. - $ curl http://people.csail.mit.edu/jrennie/20Newsgroups/20news-bydate.tar.gz + $ curl http://people.csail.mit.edu/jrennie/20Newsgroups/20news-bydate.tar.gz -o ${WORK_DIR}/20news-bydate.tar.gz $ mkdir -p ${WORK_DIR}/20news-bydate $ cd ${WORK_DIR}/20news-bydate && tar xzf ../20news-bydate.tar.gz && cd .. && cd .. $ mkdir ${WORK_DIR}/20news-all $ cp -R ${WORK_DIR}/20news-bydate/*/* ${WORK_DIR}/20news-all * If you're running on a Hadoop cluster: - + $ hadoop dfs -put ${WORK_DIR}/20news-all ${WORK_DIR}/20news-all -3. Convert the full 20 newsgroups dataset into a < Text, Text > SequenceFile. - - $ mahout seqdirectory - -i ${WORK_DIR}/20news-all - -o ${WORK_DIR}/20news-seq +3. Convert the full 20 newsgroups dataset into a < Text, Text > SequenceFile. + + $ mahout seqdirectory + -i ${WORK_DIR}/20news-all + -o ${WORK_DIR}/20news-seq -ow - -4. Convert and preprocesses the dataset into a < Text, VectorWritable > SequenceFile containing term frequencies for each document. - - $ mahout seq2sparse - -i ${WORK_DIR}/20news-seq + +4. Convert and preprocesses the dataset into a < Text, VectorWritable > SequenceFile containing term frequencies for each document. + + $ mahout seq2sparse + -i ${WORK_DIR}/20news-seq -o ${WORK_DIR}/20news-vectors - -lnorm - -nv + -lnorm + -nv -wt tfidf -If we wanted to use different parsing methods or transformations on the term frequency vectors we could supply different options here e.g.: -ng 2 for bigrams or -n 2 for L2 length normalization. See the [Creating vectors from text](http://mahout.apache.org/users/basics/creating-vectors-from-text.html) page for a list of all seq2sparse options. +If we wanted to use different parsing methods or transformations on the term frequency vectors we could supply different options here e.g.: -ng 2 for bigrams or -n 2 for L2 length normalization. See the [Creating vectors from text](http://mahout.apache.org/users/basics/creating-vectors-from-text.html) page for a list of all seq2sparse options. 5. Split the preprocessed dataset into training and testing sets. - $ mahout split - -i ${WORK_DIR}/20news-vectors/tfidf-vectors - --trainingOutput ${WORK_DIR}/20news-train-vectors - --testOutput ${WORK_DIR}/20news-test-vectors - --randomSelectionPct 40 + $ mahout split + -i ${WORK_DIR}/20news-vectors/tfidf-vectors + --trainingOutput ${WORK_DIR}/20news-train-vectors + --testOutput ${WORK_DIR}/20news-test-vectors + --randomSelectionPct 40 --overwrite --sequenceFiles -xm sequential - + 6. Train the classifier. - $ mahout trainnb + $ mahout trainnb -i ${WORK_DIR}/20news-train-vectors - -el - -o ${WORK_DIR}/model - -li ${WORK_DIR}/labelindex - -ow + -el + -o ${WORK_DIR}/model + -li ${WORK_DIR}/labelindex + -ow -c 7. Test the classifier. - $ mahout testnb + $ mahout testnb -i ${WORK_DIR}/20news-test-vectors - -m ${WORK_DIR}/model - -l ${WORK_DIR}/labelindex - -ow - -o ${WORK_DIR}/20news-testing + -m ${WORK_DIR}/model + -l ${WORK_DIR}/labelindex + -ow + -o ${WORK_DIR}/20news-testing -c - - diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/classification/wikipedia-classifier-example.md b/website/Map_Reduce_Folder/tutorials/map-reduce/classification/wikipedia-classifier-example.md index d255d1f375..5a5065ec5b 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/classification/wikipedia-classifier-example.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/classification/wikipedia-classifier-example.md @@ -10,7 +10,7 @@ redirect_from: # Wikipedia XML parser and Naive Bayes Classifier Example ## Introduction -Mahout has an [example script](https://github.com/apache/mahout/blob/master/examples/bin/classify-wikipedia.sh) [1] which will download a recent XML dump of the (entire if desired) [English Wikipedia database](http://dumps.wikimedia.org/enwiki/latest/). After running the classification script, you can use the [document classification script](https://github.com/apache/mahout/blob/master/examples/bin/spark-document-classifier.mscala) from the Mahout [spark-shell](http://mahout.apache.org/users/sparkbindings/play-with-shell.html) to vectorize and classify text from outside of the training and testing corpus using a modle built on the Wikipedia dataset. +Mahout has an [example script](https://github.com/apache/mahout/blob/master/examples/bin/classify-wikipedia.sh) [1] which will download a recent XML dump of the (entire if desired) [English Wikipedia database](http://dumps.wikimedia.org/enwiki/latest/). After running the classification script, you can use the [document classification script](https://github.com/apache/mahout/blob/master/examples/bin/spark-document-classifier.mscala) from the Mahout [spark-shell](http://mahout.apache.org/users/sparkbindings/play-with-shell.html) to vectorize and classify text from outside of the training and testing corpus using a modle built on the Wikipedia dataset. You can run this script to build and test a Naive Bayes classifier for option (1) 10 arbitrary countries or option (2) 2 countries (United States and United Kingdom). @@ -22,7 +22,7 @@ By defult the script is set to run on a medium sized Wikipedia XML dump. To run The step by step process for Creating a Naive Bayes Classifier for the Wikipedia XML dump is very similar to that for [creating a 20 Newsgroups Classifier](http://mahout.apache.org/users/classification/twenty-newsgroups.html) [4]. The only difference being that instead of running `$mahout seqdirectory` on the unzipped 20 Newsgroups file, you'll run `$mahout seqwiki` on the unzipped Wikipedia xml dump. - $ mahout seqwiki + $ mahout seqwiki The above command launches `WikipediaToSequenceFile.java` which accepts a text file of categories [3] and starts an MR job to parse the each document in the XML file. This process will seek to extract documents with a wikipedia category tag which (exactly, if the `-exactMatchOnly` option is set) matches a line in the category file. If no match is found and the `-all` option is set, the document will be dumped into an "unknown" category. The documents will then be written out as a `` sequence file of the form (K:/category/document_title , V: document). @@ -57,4 +57,3 @@ After `seqwiki`, the script runs `seq2sparse`, `split`, `trainnb` and `testnb` a [6] [Mahout Spark Naive Bayes](http://mahout.apache.org/users/algorithms/spark-naive-bayes.html) [7] [Mahout Scala Spark and H2O Bindings](http://mahout.apache.org/users/sparkbindings/home.html) - diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/canopy-commandline.md b/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/canopy-commandline.md index e965f3f7f7..db1af2e740 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/canopy-commandline.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/canopy-commandline.md @@ -31,7 +31,7 @@ job will be mahout-core-0.3.job ## Testing it on one single machine w/o cluster * Put the data: cp testdata -* Run the Job: +* Run the Job: ./bin/mahout canopy -i testdata -o output -dm org.apache.mahout.common.distance.CosineDistanceMeasure -ow -t1 5 -t2 2 @@ -42,7 +42,7 @@ org.apache.mahout.common.distance.CosineDistanceMeasure -ow -t1 5 -t2 2 * (As needed) Start up Hadoop: $HADOOP_HOME/bin/start-all.sh * Put the data: $HADOOP_HOME/bin/hadoop fs -put testdata -* Run the Job: +* Run the Job: export HADOOP_HOME= export HADOOP_CONF_DIR=$HADOOP_HOME/conf @@ -55,18 +55,17 @@ to view all outputs. # Command line options - --input (-i) input Path to job input directory.Must - be a SequenceFile of - VectorWritable - --output (-o) output The directory pathname for output. - --overwrite (-ow) If present, overwrite the output - directory before running job - --distanceMeasure (-dm) distanceMeasure The classname of the - DistanceMeasure. Default is - SquaredEuclidean - --t1 (-t1) t1 T1 threshold value - --t2 (-t2) t2 T2 threshold value - --clustering (-cl) If present, run clustering after - the iterations have taken place - --help (-h) Print out help - + --input (-i) input Path to job input directory.Must + be a SequenceFile of + VectorWritable + --output (-o) output The directory pathname for output. + --overwrite (-ow) If present, overwrite the output + directory before running job + --distanceMeasure (-dm) distanceMeasure The classname of the + DistanceMeasure. Default is + SquaredEuclidean + --t1 (-t1) t1 T1 threshold value + --t2 (-t2) t2 T2 threshold value + --clustering (-cl) If present, run clustering after + the iterations have taken place + --help (-h) Print out help diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/clustering-of-synthetic-control-data.md b/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/clustering-of-synthetic-control-data.md index 9d5912397d..ce4dc580e2 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/clustering-of-synthetic-control-data.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/clustering-of-synthetic-control-data.md @@ -13,13 +13,13 @@ redirect_from: This example will demonstrate clustering of time series data, specifically control charts. [Control charts](http://en.wikipedia.org/wiki/Control_chart) are tools used to determine whether a manufacturing or business process is in a state of statistical control. Such control charts are generated / simulated repeatedly at equal time intervals. A [simulated dataset](http://archive.ics.uci.edu/ml/databases/synthetic_control/synthetic_control.data.html) is available for use in UCI machine learning repository. -A time series of control charts needs to be clustered into their close knit groups. The data set we use is synthetic and is meant to resemble real world information in an anonymized format. It contains six different classes: Normal, Cyclic, Increasing trend, Decreasing trend, Upward shift, Downward shift. In this example we will use Mahout to cluster the data into corresponding class buckets. +A time series of control charts needs to be clustered into their close knit groups. The data set we use is synthetic and is meant to resemble real world information in an anonymized format. It contains six different classes: Normal, Cyclic, Increasing trend, Decreasing trend, Upward shift, Downward shift. In this example we will use Mahout to cluster the data into corresponding class buckets. *For the sake of simplicity, we won't use a cluster in this example, but instead show you the commands to run the clustering examples locally with Hadoop*. ## Setup -We need to do some initial setup before we are able to run the example. +We need to do some initial setup before we are able to run the example. 1. Start out by downloading the dataset to be clustered from the UCI Machine Learning Repository: [http://archive.ics.uci.edu/ml/databases/synthetic_control/synthetic_control.data](http://archive.ics.uci.edu/ml/databases/synthetic_control/synthetic_control.data). @@ -52,4 +52,3 @@ Depending on the clustering algorithm you want to run, the following commands ca bin/mahout org.apache.mahout.clustering.syntheticcontrol.fuzzykmeans.Job The clustering output will be produced in the *output* directory. The output data points are in vector format. In order to read/analyze the output, you can use the [clusterdump](/users/clustering/cluster-dumper.html) utility provided by Mahout. - diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/clusteringyourdata.md b/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/clusteringyourdata.md index ef74a61b00..3a005aca56 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/clusteringyourdata.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/clusteringyourdata.md @@ -49,34 +49,34 @@ Mahout has a cluster dumper utility that can be used to retrieve and evaluate yo ## The cluster dumper options are: - --help (-h) Print out help - - --input (-i) input The directory containing Sequence - Files for the Clusters + --help (-h) Print out help - --output (-o) output The output file. If not specified, + --input (-i) input The directory containing Sequence + Files for the Clusters + + --output (-o) output The output file. If not specified, dumps to the console. --outputFormat (-of) outputFormat The optional output format to write - the results as. Options: TEXT, CSV, or GRAPH_ML + the results as. Options: TEXT, CSV, or GRAPH_ML + + --substring (-b) substring The number of chars of the + asFormatString() to print - --substring (-b) substring The number of chars of the - asFormatString() to print - - --pointsDir (-p) pointsDir The directory containing points - sequence files mapping input vectors to their cluster. If specified, - then the program will output the - points associated with a cluster + --pointsDir (-p) pointsDir The directory containing points + sequence files mapping input vectors to their cluster. If specified, + then the program will output the + points associated with a cluster - --dictionary (-d) dictionary The dictionary file. + --dictionary (-d) dictionary The dictionary file. - --dictionaryType (-dt) dictionaryType The dictionary file type + --dictionaryType (-dt) dictionaryType The dictionary file type (text|sequencefile) --distanceMeasure (-dm) distanceMeasure The classname of the DistanceMeasure. - Default is SquaredEuclidean. + Default is SquaredEuclidean. - --numWords (-n) numWords The number of top terms to print + --numWords (-n) numWords The number of top terms to print --tempDir tempDir Intermediate output directory @@ -86,7 +86,7 @@ Mahout has a cluster dumper utility that can be used to retrieve and evaluate yo --evaluate (-e) Run ClusterEvaluator and CDbwEvaluator over the input. The output will be appended to the rest of - the output at the end. + the output at the end. More information on using clusterdump utility can be found [here](cluster-dumper.html) @@ -104,7 +104,7 @@ centroid adjusted by the log_2 of the number of clusters. This can be compared to the squared magnitude of the original data or the squared deviation from the centroid for all of the data. The idea is that you are changing the representation of the data by allocating some of the bits in -your original representation to represent which cluster each point is in. +your original representation to represent which cluster each point is in. If those bits aren't made up by the residue being small then your clustering is making a bad trade-off. @@ -125,4 +125,3 @@ Unfortunately, it is pretty easy to kid yourself into thinking your system is working using this kind of inspection. The problem is that we are too good at seeing (making up) patterns. {quote} - diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/fuzzy-k-means-commandline.md b/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/fuzzy-k-means-commandline.md index df555fedd1..9bf8f9d1e1 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/fuzzy-k-means-commandline.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/fuzzy-k-means-commandline.md @@ -32,7 +32,7 @@ job will be mahout-core-0.3.job ## Testing it on one single machine w/o cluster * Put the data: cp testdata -* Run the Job: +* Run the Job: ./bin/mahout fkmeans -i testdata @@ -42,7 +42,7 @@ job will be mahout-core-0.3.job * (As needed) Start up Hadoop: $HADOOP_HOME/bin/start-all.sh * Put the data: $HADOOP_HOME/bin/hadoop fs -put testdata -* Run the Job: +* Run the Job: export HADOOP_HOME= export HADOOP_CONF_DIR=$HADOOP_HOME/conf @@ -54,47 +54,45 @@ to view all outputs. # Command line options - --input (-i) input Path to job input directory. - Must be a SequenceFile of - VectorWritable - --clusters (-c) clusters The input centroids, as Vectors. - Must be a SequenceFile of - Writable, Cluster/Canopy. If k - is also specified, then a random - set of vectors will be selected - and written out to this path - first - --output (-o) output The directory pathname for - output. - --distanceMeasure (-dm) distanceMeasure The classname of the - DistanceMeasure. Default is - SquaredEuclidean - --convergenceDelta (-cd) convergenceDelta The convergence delta value. - Default is 0.5 - --maxIter (-x) maxIter The maximum number of - iterations. - --k (-k) k The k in k-Means. If specified, - then a random selection of k + --input (-i) input Path to job input directory. + Must be a SequenceFile of + VectorWritable + --clusters (-c) clusters The input centroids, as Vectors. + Must be a SequenceFile of + Writable, Cluster/Canopy. If k + is also specified, then a random + set of vectors will be selected + and written out to this path + first + --output (-o) output The directory pathname for + output. + --distanceMeasure (-dm) distanceMeasure The classname of the + DistanceMeasure. Default is + SquaredEuclidean + --convergenceDelta (-cd) convergenceDelta The convergence delta value. + Default is 0.5 + --maxIter (-x) maxIter The maximum number of + iterations. + --k (-k) k The k in k-Means. If specified, + then a random selection of k Vectors will be chosen as the - Centroid and written to the - clusters input path. - --m (-m) m coefficient normalization - factor, must be greater than 1 - --overwrite (-ow) If present, overwrite the output - directory before running job - --help (-h) Print out help - --numMap (-u) numMap The number of map tasks. - Defaults to 10 - --maxRed (-r) maxRed The number of reduce tasks. - Defaults to 2 - --emitMostLikely (-e) emitMostLikely True if clustering should emit - the most likely point only, - false for threshold clustering. - Default is true - --threshold (-t) threshold The pdf threshold used for - cluster determination. Default - is 0 - --clustering (-cl) If present, run clustering after - the iterations have taken place - - + Centroid and written to the + clusters input path. + --m (-m) m coefficient normalization + factor, must be greater than 1 + --overwrite (-ow) If present, overwrite the output + directory before running job + --help (-h) Print out help + --numMap (-u) numMap The number of map tasks. + Defaults to 10 + --maxRed (-r) maxRed The number of reduce tasks. + Defaults to 2 + --emitMostLikely (-e) emitMostLikely True if clustering should emit + the most likely point only, + false for threshold clustering. + Default is true + --threshold (-t) threshold The pdf threshold used for + cluster determination. Default + is 0 + --clustering (-cl) If present, run clustering after + the iterations have taken place diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/k-means-commandline.md b/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/k-means-commandline.md index e6c92a1e66..3543146988 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/k-means-commandline.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/k-means-commandline.md @@ -12,7 +12,7 @@ redirect_from: # kMeans commandline introduction This quick start page describes how to run the kMeans clustering algorithm -on a Hadoop cluster. +on a Hadoop cluster. # Steps @@ -39,7 +39,7 @@ job will be mahout-core-0.3.job ## Testing it on one single machine w/o cluster * Put the data: cp testdata -* Run the Job: +* Run the Job: ./bin/mahout kmeans -i testdata -o output -c clusters -dm org.apache.mahout.common.distance.CosineDistanceMeasure -x 5 -ow -cd 1 -k @@ -51,7 +51,7 @@ org.apache.mahout.common.distance.CosineDistanceMeasure -x 5 -ow -cd 1 -k * (As needed) Start up Hadoop: $HADOOP_HOME/bin/start-all.sh * Put the data: $HADOOP_HOME/bin/hadoop fs -put testdata -* Run the Job: +* Run the Job: export HADOOP_HOME= export HADOOP_CONF_DIR=$HADOOP_HOME/conf @@ -63,35 +63,34 @@ to view all outputs. # Command line options - --input (-i) input Path to job input directory. - Must be a SequenceFile of - VectorWritable - --clusters (-c) clusters The input centroids, as Vectors. - Must be a SequenceFile of - Writable, Cluster/Canopy. If k - is also specified, then a random - set of vectors will be selected - and written out to this path - first - --output (-o) output The directory pathname for - output. - --distanceMeasure (-dm) distanceMeasure The classname of the - DistanceMeasure. Default is - SquaredEuclidean - --convergenceDelta (-cd) convergenceDelta The convergence delta value. - Default is 0.5 - --maxIter (-x) maxIter The maximum number of - iterations. - --maxRed (-r) maxRed The number of reduce tasks. - Defaults to 2 - --k (-k) k The k in k-Means. If specified, - then a random selection of k - Vectors will be chosen as the - Centroid and written to the - clusters input path. - --overwrite (-ow) If present, overwrite the output - directory before running job - --help (-h) Print out help - --clustering (-cl) If present, run clustering after - the iterations have taken place - + --input (-i) input Path to job input directory. + Must be a SequenceFile of + VectorWritable + --clusters (-c) clusters The input centroids, as Vectors. + Must be a SequenceFile of + Writable, Cluster/Canopy. If k + is also specified, then a random + set of vectors will be selected + and written out to this path + first + --output (-o) output The directory pathname for + output. + --distanceMeasure (-dm) distanceMeasure The classname of the + DistanceMeasure. Default is + SquaredEuclidean + --convergenceDelta (-cd) convergenceDelta The convergence delta value. + Default is 0.5 + --maxIter (-x) maxIter The maximum number of + iterations. + --maxRed (-r) maxRed The number of reduce tasks. + Defaults to 2 + --k (-k) k The k in k-Means. If specified, + then a random selection of k + Vectors will be chosen as the + Centroid and written to the + clusters input path. + --overwrite (-ow) If present, overwrite the output + directory before running job + --help (-h) Print out help + --clustering (-cl) If present, run clustering after + the iterations have taken place diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/lda-commandline.md b/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/lda-commandline.md index 73e188808b..6edfec5f97 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/lda-commandline.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/lda-commandline.md @@ -10,7 +10,7 @@ redirect_from: # Running Latent Dirichlet Allocation (algorithm) from the Command Line [Since Mahout v0.6](https://issues.apache.org/jira/browse/MAHOUT-897) - lda has been implemented as Collapsed Variable Bayes (cvb). + lda has been implemented as Collapsed Variable Bayes (cvb). Mahout's LDA can be launched from the same command line invocation whether you are running on a single machine in stand-alone mode or on a larger @@ -35,7 +35,7 @@ job will be mahout-core-0.3.job ## Testing it on one single machine w/o cluster * Put the data: cp testdata -* Run the Job: +* Run the Job: ./bin/mahout cvb -i testdata @@ -45,7 +45,7 @@ job will be mahout-core-0.3.job * (As needed) Start up Hadoop: $HADOOP_HOME/bin/start-all.sh * Put the data: $HADOOP_HOME/bin/hadoop fs -put testdata -* Run the Job: +* Run the Job: export HADOOP_HOME= export HADOOP_CONF_DIR=$HADOOP_HOME/conf @@ -57,29 +57,28 @@ to view all outputs. # Command line options from Mahout cvb version 0.8 - mahout cvb -h - --input (-i) input Path to job input directory. - --output (-o) output The directory pathname for output. - --maxIter (-x) maxIter The maximum number of iterations. - --convergenceDelta (-cd) convergenceDelta The convergence delta value - --overwrite (-ow) If present, overwrite the output directory before running job - --num_topics (-k) num_topics Number of topics to learn - --num_terms (-nt) num_terms Vocabulary size - --doc_topic_smoothing (-a) doc_topic_smoothing Smoothing for document/topic distribution - --term_topic_smoothing (-e) term_topic_smoothing Smoothing for topic/term distribution - --dictionary (-dict) dictionary Path to term-dictionary file(s) (glob expression supported) - --doc_topic_output (-dt) doc_topic_output Output path for the training doc/topic distribution - --topic_model_temp_dir (-mt) topic_model_temp_dir Path to intermediate model path (useful for restarting) - --iteration_block_size (-block) iteration_block_size Number of iterations per perplexity check - --random_seed (-seed) random_seed Random seed - --test_set_fraction (-tf) test_set_fraction Fraction of data to hold out for testing - --num_train_threads (-ntt) num_train_threads number of threads per mapper to train with - --num_update_threads (-nut) num_update_threads number of threads per mapper to update the model with - --max_doc_topic_iters (-mipd) max_doc_topic_iters max number of iterations per doc for p(topic|doc) learning - --num_reduce_tasks num_reduce_tasks number of reducers to use during model estimation - --backfill_perplexity enable backfilling of missing perplexity values - --help (-h) Print out help - --tempDir tempDir Intermediate output directory - --startPhase startPhase First phase to run + mahout cvb -h + --input (-i) input Path to job input directory. + --output (-o) output The directory pathname for output. + --maxIter (-x) maxIter The maximum number of iterations. + --convergenceDelta (-cd) convergenceDelta The convergence delta value + --overwrite (-ow) If present, overwrite the output directory before running job + --num_topics (-k) num_topics Number of topics to learn + --num_terms (-nt) num_terms Vocabulary size + --doc_topic_smoothing (-a) doc_topic_smoothing Smoothing for document/topic distribution + --term_topic_smoothing (-e) term_topic_smoothing Smoothing for topic/term distribution + --dictionary (-dict) dictionary Path to term-dictionary file(s) (glob expression supported) + --doc_topic_output (-dt) doc_topic_output Output path for the training doc/topic distribution + --topic_model_temp_dir (-mt) topic_model_temp_dir Path to intermediate model path (useful for restarting) + --iteration_block_size (-block) iteration_block_size Number of iterations per perplexity check + --random_seed (-seed) random_seed Random seed + --test_set_fraction (-tf) test_set_fraction Fraction of data to hold out for testing + --num_train_threads (-ntt) num_train_threads number of threads per mapper to train with + --num_update_threads (-nut) num_update_threads number of threads per mapper to update the model with + --max_doc_topic_iters (-mipd) max_doc_topic_iters max number of iterations per doc for p(topic|doc) learning + --num_reduce_tasks num_reduce_tasks number of reducers to use during model estimation + --backfill_perplexity enable backfilling of missing perplexity values + --help (-h) Print out help + --tempDir tempDir Intermediate output directory + --startPhase startPhase First phase to run --endPhase endPhase Last phase to run - diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/viewing-results.md b/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/viewing-results.md index d2cca70ef3..e36cbe584c 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/viewing-results.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/viewing-results.md @@ -42,7 +42,7 @@ Run the following to print out all options: ./solr-clust-n2/out/clusters-2 --dictionary ./solr-clust-n2/dictionary.txt --substring 100 --pointsDir ./solr-clust-n2/out/points/ - + diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/visualizing-sample-clusters.md b/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/visualizing-sample-clusters.md index 4ef2c0482c..1ef8179287 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/visualizing-sample-clusters.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/clustering/visualizing-sample-clusters.md @@ -43,7 +43,7 @@ If you are using Eclipse, just right-click on each of the classes mentioned abov cd $MAHOUT_HOME/examples mvn -q exec:java -Dexec.mainClass=org.apache.mahout.clustering.display.DisplayClustering -You can substitute other names above for *DisplayClustering*. +You can substitute other names above for *DisplayClustering*. Note that some of these programs display the sample points and then superimpose all of the clusters from each iteration. The last iteration's clusters are in diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/misc/parallel-frequent-pattern-mining.md b/website/Map_Reduce_Folder/tutorials/map-reduce/misc/parallel-frequent-pattern-mining.md index 090410080e..f7bbfac13c 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/misc/parallel-frequent-pattern-mining.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/misc/parallel-frequent-pattern-mining.md @@ -73,7 +73,7 @@ frequently occurring sets of items and their counts from the given input data $MAHOUT_HOME/core/src/test/resources/retail.dat is a sample dataset in this -format. +format. Other sample files are accident.dat.gz from [http://fimi.cs.helsinki.fi/data/](http://fimi.cs.helsinki.fi/data/) . As a quick test, try this: @@ -183,6 +183,6 @@ of the Frequent Patterns: -n 4 or replace -n 4 with -c for the count of patterns. - + Open questions: how does one experiment and monitor with these various parameters? diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/misc/testing.md b/website/Map_Reduce_Folder/tutorials/map-reduce/misc/testing.md index e459e16b8d..7038e10c3a 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/misc/testing.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/misc/testing.md @@ -28,13 +28,13 @@ Hadoop jobs. ## Regressions/Benchmarks/Integrations * Algorithmic quality and speed are not tested, except in a few instances. Such tests often require much longer run times (minutes to hours), a -running Hadoop cluster, and downloads of large datasets (in the megabytes). -* Standardized speed tests are difficult on different hardware. +running Hadoop cluster, and downloads of large datasets (in the megabytes). +* Standardized speed tests are difficult on different hardware. * Unit tests of external integrations require access to externals: HDFS, -S3, JDBC, Cassandra, etc. +S3, JDBC, Cassandra, etc. Apache Jenkins is not able to support these environments. Commercial -donations would help. +donations would help. ## Unit Tests @@ -46,4 +46,3 @@ of a great many utilities and algorithms. A much smaller percent include ## Other - diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/misc/using-mahout-with-python-via-jpype.md b/website/Map_Reduce_Folder/tutorials/map-reduce/misc/using-mahout-with-python-via-jpype.md index f5ce2e0374..50e84e03d4 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/misc/using-mahout-with-python-via-jpype.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/misc/using-mahout-with-python-via-jpype.md @@ -66,20 +66,20 @@ vectors from two Gaussian distributions with unit variance. """Create a sequence file containing some normally distributed ifile - path to the sequence file to create """ - + #matrix of the cluster means cmeans=np.array([[1,1] ,[-1,-1]],np.int) - + nperc=30 #number of points per cluster - + vecs=[] - + vnames=[] for cind in range(cmeans.shape[0]): pts=np.random.randn(nperc,2) pts=pts+cmeans[cind,:].reshape([1,cmeans.shape[1]]) vecs.append(pts) - + #names for the vectors #names are just the points with an index #we do this so we can validate by cross-refencing the name with thevector @@ -87,43 +87,43 @@ vectors from two Gaussian distributions with unit variance. for row in range(nperc): vn[row]="c"+str(cind)+"_"+pts[row,0].astype((np.str,4))+"_"+pts[row,1].astype((np.str,4)) vnames.append(vn) - + vecs=np.vstack(vecs) vnames=np.hstack(vnames) - - + + #start the jvm start_jpype() - + #create the sequence file that we will write to - io=JPackage("org").apache.hadoop.io + io=JPackage("org").apache.hadoop.io FileSystemCls=JPackage("org").apache.hadoop.fs.FileSystem - + PathCls=JPackage("org").apache.hadoop.fs.Path path=PathCls(ifile) - - ConfCls=JPackage("org").apache.hadoop.conf.Configuration + + ConfCls=JPackage("org").apache.hadoop.conf.Configuration conf=ConfCls() - + fs=FileSystemCls.get(conf) - + #vector classes VectorWritableCls=JPackage("org").apache.mahout.math.VectorWritable DenseVectorCls=JPackage("org").apache.mahout.math.DenseVector NamedVectorCls=JPackage("org").apache.mahout.math.NamedVector writer=io.SequenceFile.createWriter(fs, conf, path,io.Text,VectorWritableCls) - - + + vecwritable=VectorWritableCls() for row in range(vecs.shape[0]): nvector=NamedVectorCls(DenseVectorCls(JArray(JDouble,1)(vecs[row,:])),vnames[row]) #need to wrap key and value because of overloading wrapkey=JObject(io.Text("key "+str(row)),io.Writable) wrapval=JObject(vecwritable,io.Writable) - + vecwritable.set(nvector) writer.append(wrapkey,wrapval) - + writer.close() @@ -135,43 +135,43 @@ mahout. def read_clustered_pts(ifile,*args,**param): """Read the clustered points ifile - path to the sequence file containing the clustered points - """ - + """ + #start the jvm start_jpype() - + #create the sequence file that we will write to - io=JPackage("org").apache.hadoop.io + io=JPackage("org").apache.hadoop.io FileSystemCls=JPackage("org").apache.hadoop.fs.FileSystem - + PathCls=JPackage("org").apache.hadoop.fs.Path path=PathCls(ifile) - - ConfCls=JPackage("org").apache.hadoop.conf.Configuration + + ConfCls=JPackage("org").apache.hadoop.conf.Configuration conf=ConfCls() - + fs=FileSystemCls.get(conf) - + #vector classes VectorWritableCls=JPackage("org").apache.mahout.math.VectorWritable NamedVectorCls=JPackage("org").apache.mahout.math.NamedVector - - - ReaderCls=io.__getattribute__("SequenceFile$Reader") + + + ReaderCls=io.__getattribute__("SequenceFile$Reader") reader=ReaderCls(fs, path,conf) - - + + key=reader.getKeyClass()() - - + + valcls=reader.getValueClass() vecwritable=valcls() - while (reader.next(key,vecwritable)): + while (reader.next(key,vecwritable)): weight=vecwritable.getWeight() nvec=vecwritable.getVector() - + cname=nvec.__class__.__name__ - if (cname.rsplit('.',1)[1]=="NamedVector"): + if (cname.rsplit('.',1)[1]=="NamedVector"): print "cluster={key} Name={name} x={x}y={y}".format(key=key.toString(),name=nvec.getName(),x=nvec.get(0),y=nvec.get(1)) else: raise NotImplementedError("Vector isn't a NamedVector. Need tomodify/test the code to handle this case.") @@ -185,40 +185,39 @@ found by mahout, def getClusters(ifile,*args,**param): """Read the centroids from the clusters outputted by kmenas ifile - Path to the sequence file containing the centroids - """ - + """ + #start the jvm start_jpype() - + #create the sequence file that we will write to - io=JPackage("org").apache.hadoop.io + io=JPackage("org").apache.hadoop.io FileSystemCls=JPackage("org").apache.hadoop.fs.FileSystem - + PathCls=JPackage("org").apache.hadoop.fs.Path path=PathCls(ifile) - - ConfCls=JPackage("org").apache.hadoop.conf.Configuration + + ConfCls=JPackage("org").apache.hadoop.conf.Configuration conf=ConfCls() - + fs=FileSystemCls.get(conf) - + #vector classes VectorWritableCls=JPackage("org").apache.mahout.math.VectorWritable NamedVectorCls=JPackage("org").apache.mahout.math.NamedVector ReaderCls=io.__getattribute__("SequenceFile$Reader") reader=ReaderCls(fs, path,conf) - - + + key=io.Text() - - + + valcls=reader.getValueClass() - + vecwritable=valcls() - - while (reader.next(key,vecwritable)): + + while (reader.next(key,vecwritable)): center=vecwritable.getCenter() - + print "id={cid}center={center}".format(cid=vecwritable.getId(),center=center.values) pass - diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/intro-als-hadoop.md b/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/intro-als-hadoop.md index af5d401a52..b7f1b89c7f 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/intro-als-hadoop.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/intro-als-hadoop.md @@ -11,11 +11,11 @@ redirect_from: ##Overview -Mahout’s ALS recommender is a matrix factorization algorithm that uses Alternating Least Squares with Weighted-Lamda-Regularization (ALS-WR). It factors the user to item matrix *A* into the user-to-feature matrix *U* and the item-to-feature matrix *M*: It runs the ALS algorithm in a parallel fashion. The algorithm details can be referred to in the following papers: +Mahout’s ALS recommender is a matrix factorization algorithm that uses Alternating Least Squares with Weighted-Lamda-Regularization (ALS-WR). It factors the user to item matrix *A* into the user-to-feature matrix *U* and the item-to-feature matrix *M*: It runs the ALS algorithm in a parallel fashion. The algorithm details can be referred to in the following papers: * [Large-scale Parallel Collaborative Filtering for the Netflix Prize](http://www.hpl.hp.com/personal/Robert_Schreiber/papers/2008%20AAIM%20Netflix/netflix_aaim08%28submitted%29.pdf) -* [Collaborative Filtering for Implicit Feedback Datasets](http://research.yahoo.com/pub/2433) +* [Collaborative Filtering for Implicit Feedback Datasets](http://research.yahoo.com/pub/2433) This recommendation algorithm can be used in eCommerce platform to recommend products to customers. Unlike the user or item based recommenders that computes the similarity of users or items to make recommendations, the ALS algorithm uncovers the latent factors that explain the observed user to item ratings and tries to find optimal factor weights to minimize the least squares between predicted and actual ratings. @@ -25,7 +25,7 @@ One of the strengths of the ALS based recommender, compared to the user or item ##Implementation At present Mahout has a map-reduce implementation of ALS, which is composed of 2 jobs: a parallel matrix factorization job and a recommendation job. -The matrix factorization job computes the user-to-feature matrix and item-to-feature matrix given the user to item ratings. Its input includes: +The matrix factorization job computes the user-to-feature matrix and item-to-feature matrix given the user to item ratings. Its input includes:
     --input: directory containing files of explicit user to item rating or implicit feedback;
     --output: output path of the user-feature matrix and feature-item matrix;
@@ -37,7 +37,7 @@ The matrix factorization job computes the user-to-feature matrix and item-to-fea
     --numIterations: number of iterations
     --usesLongIDs: boolean flag to indicate whether the input contains long IDs that need to be translated
 
-and it outputs the matrices in sequence file format. +and it outputs the matrices in sequence file format. The recommendation job uses the user feature matrix and item feature matrix calculated from the factorization job to compute the top-N recommendations per user. Its input includes:
@@ -50,9 +50,9 @@ The recommendation job uses the user feature matrix and item feature matrix calc
     --numThreads: number of threads per mapper;
     --usesLongIDs: boolean flag to indicate whether the input contains long IDs that need to be translated;
     --userIDIndex: index for user long IDs (necessary if usesLongIDs is true);
-    --itemIDIndex: index for item long IDs (necessary if usesLongIDs is true) 
+    --itemIDIndex: index for item long IDs (necessary if usesLongIDs is true)
 
-and it outputs a list of recommended item ids for each user. The predicted rating between user and item is a dot product of the user's feature vector and the item's feature vector. +and it outputs a list of recommended item ids for each user. The predicted rating between user and item is a dot product of the user's feature vector and the item's feature vector. ##Example @@ -77,13 +77,13 @@ To quickly start, you could specify a text file like following as the input: **Step 2: Determine parameters** -In addition, users need to determine dimension of feature space, the number of iterations to run the alternating least square algorithm, Using 10 features and 15 iterations is a reasonable default to try first. Optionally a confidence parameter can be set if the input preference is implicit user feedback. +In addition, users need to determine dimension of feature space, the number of iterations to run the alternating least square algorithm, Using 10 features and 15 iterations is a reasonable default to try first. Optionally a confidence parameter can be set if the input preference is implicit user feedback. **Step 3: Run ALS** Assuming your *JAVA_HOME* is appropriately set and Mahout was installed properly we’re ready to configure our syntax. Enter the following command: - $ mahout parallelALS --input $als_input --output $als_output --lambda 0.1 --implicitFeedback true --alpha 0.8 --numFeatures 2 --numIterations 5 --numThreadsPerSolver 1 --tempDir tmp + $ mahout parallelALS --input $als_input --output $als_output --lambda 0.1 --implicitFeedback true --alpha 0.8 --numFeatures 2 --numIterations 5 --numThreadsPerSolver 1 --tempDir tmp Running the command will execute a series of jobs the final product of which will be an output file deposited to the output directory specified in the command syntax. The output directory contains 3 sub-directories: *M* stores the item to feature matrix, *U* stores the user to feature matrix and userRatings stores the user's ratings on the items. The *tempDir* parameter specifies the directory to store the intermediate output of the job, such as the matrix output in each iteration and each item's average rating. Using the *tempDir* will help on debugging. @@ -93,8 +93,7 @@ Based on the output feature matrices from step 3, we could make recommendations $ mahout recommendfactorized --input $als_recommender_input --userFeatures $als_output/U/ --itemFeatures $als_output/M/ --numRecommendations 1 --output recommendations --maxRating 1 -The input user file is a sequence file, the sequence record key is user id and value is the user's rated item ids which will be removed from recommendation. The output file generated in our simple example will be a text file giving the recommended item ids for each user. -Remember to translate the Mahout ids back into your application specific ids. +The input user file is a sequence file, the sequence record key is user id and value is the user's rated item ids which will be removed from recommendation. The output file generated in our simple example will be a text file giving the recommended item ids for each user. +Remember to translate the Mahout ids back into your application specific ids. There exist a variety of parameters for Mahout’s ALS recommender to accommodate custom business requirements; exploring and testing various configurations to suit your needs will doubtless lead to additional questions. Feel free to ask such questions on the [mailing list](https://mahout.apache.org/community/mailing-lists,-irc-and-archives.html). - diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/intro-cooccurrence-spark.md b/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/intro-cooccurrence-spark.md index 60f3f28925..a3c33ec88f 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/intro-cooccurrence-spark.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/intro-cooccurrence-spark.md @@ -10,9 +10,9 @@ redirect_from: #Intro to Cooccurrence Recommenders with Spark -Mahout provides several important building blocks for creating recommendations using Spark. *spark-itemsimilarity* can -be used to create "other people also liked these things" type recommendations and paired with a search engine can -personalize recommendations for individual users. *spark-rowsimilarity* can provide non-personalized content based +Mahout provides several important building blocks for creating recommendations using Spark. *spark-itemsimilarity* can +be used to create "other people also liked these things" type recommendations and paired with a search engine can +personalize recommendations for individual users. *spark-rowsimilarity* can provide non-personalized content based recommendations and when paired with a search engine can be used to personalize content based recommendations. ##References @@ -28,20 +28,20 @@ Below are the command line jobs but the drivers and associated code can also be ##1. spark-itemsimilarity *spark-itemsimilarity* is the Spark counterpart of the of the Mahout mapreduce job called *itemsimilarity*. It takes in elements of interactions, which have userID, itemID, and optionally a value. It will produce one of more indicator matrices created by comparing every user's interactions with every other user. The indicator matrix is an item x item matrix where the values are log-likelihood ratio strengths. For the legacy mapreduce version, there were several possible similarity measures but these are being deprecated in favor of LLR because in practice it performs the best. -Mahout's mapreduce version of itemsimilarity takes a text file that is expected to have user and item IDs that conform to +Mahout's mapreduce version of itemsimilarity takes a text file that is expected to have user and item IDs that conform to Mahout's ID requirements--they are non-negative integers that can be viewed as row and column numbers in a matrix. -*spark-itemsimilarity* also extends the notion of cooccurrence to cross-cooccurrence, in other words the Spark version will -account for multi-modal interactions and create indicator matrices allowing the use of much more data in -creating recommendations or similar item lists. People try to do this by mixing different actions and giving them weights. +*spark-itemsimilarity* also extends the notion of cooccurrence to cross-cooccurrence, in other words the Spark version will +account for multi-modal interactions and create indicator matrices allowing the use of much more data in +creating recommendations or similar item lists. People try to do this by mixing different actions and giving them weights. For instance they might say an item-view is 0.2 of an item purchase. In practice this is often not helpful. Spark-itemsimilarity's cross-cooccurrence is a more principled way to handle this case. In effect it scrubs secondary actions with the action you want -to recommend. +to recommend. spark-itemsimilarity Mahout 1.0 Usage: spark-itemsimilarity [options] - + Disconnected from the target VM, address: '127.0.0.1:64676', transport: 'socket' Input, output options -i | --input @@ -50,15 +50,15 @@ to recommend. Secondary input path for cross-similarity calculation, same restrictions as "--input" (optional). Default: empty. -o | --output Path for output, any local or HDFS supported URI (required) - + Algorithm control options: -mppu | --maxPrefs Max number of preferences to consider per user (optional). Default: 500 -m | --maxSimilaritiesPerItem Limit the number of similarities per item to this number (optional). Default: 100 - + Note: Only the Log Likelihood Ratio (LLR) is supported as a similarity measure. - + Input text file schema options: -id | --inDelim Input delimiter character (optional). Default: "[,\t]" @@ -72,15 +72,15 @@ to recommend. Column number (0 based Int) containing the item ID string (optional). Default: 1 -fc | --filterColumn Column number (0 based Int) containing the filter string (optional). Default: -1 for no filter - + Using all defaults the input is expected of the form: "userIDitemId" or "userIDitemIDany-text..." and all rows will be used - + File discovery options: -r | --recursive Searched the -i path recursively for files that match --filenamePattern (optional), Default: false -fp | --filenamePattern Regex to match in determining input files (optional). Default: filename in the --input option or "^part-.*" if --input is a directory - + Output text file schema options: -rd | --rowKeyDelim Separates the rowID key from the vector values list (optional). Default: "\t" @@ -91,22 +91,22 @@ to recommend. -os | --omitStrength Do not write the strength to the output files (optional), Default: false. This option is used to output indexable data for creating a search engine recommender. - + Default delimiters will produce output of the form: "itemID1itemID2:value2itemID10:value10..." - + Spark config options: -ma | --master Spark Master URL (optional). Default: "local". Note that you can specify the number of cores to get a performance improvement, for example "local[4]" -sem | --sparkExecutorMem Max Java heap available as "executor memory" on each node (optional). Default: 4g -rs | --randomSeed - + -h | --help prints this usage text This looks daunting but defaults to simple fairly sane values to take exactly the same input as legacy code and is pretty flexible. It allows the user to point to a single text file, a directory full of files, or a tree of directories to be traversed recursively. The files included can be specified with either a regex-style pattern or filename. The schema for the file is defined by column numbers, which map to the important bits of data including IDs and values. The files can even contain filters, which allow unneeded rows to be discarded or used for cross-cooccurrence calculations. -See ItemSimilarityDriver.scala in Mahout's spark module if you want to customize the code. +See ItemSimilarityDriver.scala in Mahout's spark module if you want to customize the code. ###Defaults in the _**spark-itemsimilarity**_ CLI @@ -128,16 +128,16 @@ This will use the "local" Spark context and will output the standard text versio ###How To Use Multiple User Actions -Often we record various actions the user takes for later analytics. These can now be used to make recommendations. -The idea of a recommender is to recommend the action you want the user to make. For an ecom app this might be -a purchase action. It is usually not a good idea to just treat other actions the same as the action you want to recommend. -For instance a view of an item does not indicate the same intent as a purchase and if you just mixed the two together you +Often we record various actions the user takes for later analytics. These can now be used to make recommendations. +The idea of a recommender is to recommend the action you want the user to make. For an ecom app this might be +a purchase action. It is usually not a good idea to just treat other actions the same as the action you want to recommend. +For instance a view of an item does not indicate the same intent as a purchase and if you just mixed the two together you might even make worse recommendations. It is tempting though since there are so many more views than purchases. With *spark-itemsimilarity* we can now use both actions. Mahout will use cross-action cooccurrence analysis to limit the views to ones that do predict purchases. -We do this by treating the primary action (purchase) as data for the indicator matrix and use the secondary action (view) -to calculate the cross-cooccurrence indicator matrix. +We do this by treating the primary action (purchase) as data for the indicator matrix and use the secondary action (view) +to calculate the cross-cooccurrence indicator matrix. -*spark-itemsimilarity* can read separate actions from separate files or from a mixed action log by filtering certain lines. For a mixed +*spark-itemsimilarity* can read separate actions from separate files or from a mixed action log by filtering certain lines. For a mixed action log of the form: u1,purchase,iphone @@ -180,7 +180,7 @@ Use the following options: ###Output -The output of the job will be the standard text version of two Mahout DRMs. This is a case where we are calculating +The output of the job will be the standard text version of two Mahout DRMs. This is a case where we are calculating cross-cooccurrence so a primary indicator matrix and cross-cooccurrence indicator matrix will be created out-path @@ -203,7 +203,7 @@ The cross-cooccurrence indicator matrix will contain: galaxy\tnexus:1.7260924347106847 iphone:1.7260924347106847 ipad:1.7260924347106847 galaxy:1.7260924347106847 surface\tsurface:4.498681156950466 nexus:0.6795961471815897 -**Note:** You can run this multiple times to use more than two actions or you can use the underlying +**Note:** You can run this multiple times to use more than two actions or you can use the underlying SimilarityAnalysis.cooccurrence API, which will more efficiently calculate any number of cross-cooccurrence indicators. ###Log File Input @@ -229,7 +229,7 @@ A common method of storing data is in log files. If they are written using some 2014-06-23 14:46:53.115\tu3\tview\trandom text\tnexus 2014-06-23 14:46:53.115\tu4\tview\trandom text\tiphone 2014-06-23 14:46:53.115\tu4\tview\trandom text\tipad - 2014-06-23 14:46:53.115\tu4\tview\trandom text\tgalaxy + 2014-06-23 14:46:53.115\tu4\tview\trandom text\tgalaxy Can be parsed with the following CLI and run on the cluster producing the same output as the above example. @@ -246,35 +246,35 @@ Can be parsed with the following CLI and run on the cluster producing the same o ##2. spark-rowsimilarity -*spark-rowsimilarity* is the companion to *spark-itemsimilarity* the primary difference is that it takes a text file version of +*spark-rowsimilarity* is the companion to *spark-itemsimilarity* the primary difference is that it takes a text file version of a matrix of sparse vectors with optional application specific IDs and it finds similar rows rather than items (columns). Its use is -not limited to collaborative filtering. The input is in text-delimited form where there are three delimiters used. By +not limited to collaborative filtering. The input is in text-delimited form where there are three delimiters used. By default it reads (rowID<tab>columnID1:strength1<space>columnID2:strength2...) Since this job only supports LLR similarity, - which does not use the input strengths, they may be omitted in the input. It writes -(rowID<tab>rowID1:strength1<space>rowID2:strength2...) -The output is sorted by strength descending. The output can be interpreted as a row ID from the primary input followed + which does not use the input strengths, they may be omitted in the input. It writes +(rowID<tab>rowID1:strength1<space>rowID2:strength2...) +The output is sorted by strength descending. The output can be interpreted as a row ID from the primary input followed by a list of the most similar rows. The command line interface is: spark-rowsimilarity Mahout 1.0 Usage: spark-rowsimilarity [options] - + Input, output options -i | --input Input path, may be a filename, directory name, or comma delimited list of HDFS supported URIs (required) -o | --output Path for output, any local or HDFS supported URI (required) - + Algorithm control options: -mo | --maxObservations Max number of observations to consider per row (optional). Default: 500 -m | --maxSimilaritiesPerRow Limit the number of similarities per item to this number (optional). Default: 100 - + Note: Only the Log Likelihood Ratio (LLR) is supported as a similarity measure. Disconnected from the target VM, address: '127.0.0.1:49162', transport: 'socket' - + Output text file schema options: -rd | --rowKeyDelim Separates the rowID key from the vector values list (optional). Default: "\t" @@ -285,40 +285,40 @@ The command line interface is: -os | --omitStrength Do not write the strength to the output files (optional), Default: false. This option is used to output indexable data for creating a search engine recommender. - + Default delimiters will produce output of the form: "itemID1itemID2:value2itemID10:value10..." - + File discovery options: -r | --recursive Searched the -i path recursively for files that match --filenamePattern (optional), Default: false -fp | --filenamePattern Regex to match in determining input files (optional). Default: filename in the --input option or "^part-.*" if --input is a directory - + Spark config options: -ma | --master Spark Master URL (optional). Default: "local". Note that you can specify the number of cores to get a performance improvement, for example "local[4]" -sem | --sparkExecutorMem Max Java heap available as "executor memory" on each node (optional). Default: 4g -rs | --randomSeed - + -h | --help prints this usage text -See RowSimilarityDriver.scala in Mahout's spark module if you want to customize the code. +See RowSimilarityDriver.scala in Mahout's spark module if you want to customize the code. #3. Using *spark-rowsimilarity* with Text Data -Another use case for *spark-rowsimilarity* is in finding similar textual content. For instance given the tags associated with +Another use case for *spark-rowsimilarity* is in finding similar textual content. For instance given the tags associated with a blog post, - which other posts have similar tags. In this case the columns are tags and the rows are posts. Since LLR is -the only similarity method supported this is not the optimal way to determine general "bag-of-words" document similarity. -LLR is used more as a quality filter than as a similarity measure. However *spark-rowsimilarity* will produce + which other posts have similar tags. In this case the columns are tags and the rows are posts. Since LLR is +the only similarity method supported this is not the optimal way to determine general "bag-of-words" document similarity. +LLR is used more as a quality filter than as a similarity measure. However *spark-rowsimilarity* will produce lists of similar docs for every doc if input is docs with lists of terms. The Apache [Lucene](http://lucene.apache.org) project provides several methods of [analyzing and tokenizing](http://lucene.apache.org/core/4_9_0/core/org/apache/lucene/analysis/package-summary.html#package_description) documents. #4. Creating a Unified Recommender Using the output of *spark-itemsimilarity* and *spark-rowsimilarity* you can build a unified cooccurrence and content based - recommender that can be used in both or either mode depending on indicators available and the history available at + recommender that can be used in both or either mode depending on indicators available and the history available at runtime for a user. ##Requirements @@ -336,44 +336,44 @@ Indicators come in 3 types 2. **Content**: calculated from item metadata or content using *spark-rowsimilarity* 3. **Intrinsic**: assigned to items as metadata. Can be anything that describes the item. -The query for recommendations will be a mix of values meant to match one of your indicators. The query can be constructed -from user history and values derived from context (category being viewed for instance) or special precalculated data -(popularity rank for instance). This blending of indicators allows for creating many flavors or recommendations to fit +The query for recommendations will be a mix of values meant to match one of your indicators. The query can be constructed +from user history and values derived from context (category being viewed for instance) or special precalculated data +(popularity rank for instance). This blending of indicators allows for creating many flavors or recommendations to fit a very wide variety of circumstances. -With the right mix of indicators developers can construct a single query that works for completely new items and new users -while working well for items with lots of interactions and users with many recorded actions. In other words by adding in content and intrinsic +With the right mix of indicators developers can construct a single query that works for completely new items and new users +while working well for items with lots of interactions and users with many recorded actions. In other words by adding in content and intrinsic indicators developers can create a solution for the "cold-start" problem that gracefully improves with more user history -and as items have more interactions. It is also possible to create a completely content-based recommender that personalizes +and as items have more interactions. It is also possible to create a completely content-based recommender that personalizes recommendations. ##Example with 3 Indicators -You will need to decide how you store user action data so they can be processed by the item and row similarity jobs and -this is most easily done by using text files as described above. The data that is processed by these jobs is considered the -training data. You will need some amount of user history in your recs query. It is typical to use the most recent user history -but need not be exactly what is in the training set, which may include a greater volume of historical data. Keeping the user -history for query purposes could be done with a database by storing it in a users table. In the example above the two -collaborative filtering actions are "purchase" and "view", but let's also add tags (taken from catalog categories or other -descriptive metadata). +You will need to decide how you store user action data so they can be processed by the item and row similarity jobs and +this is most easily done by using text files as described above. The data that is processed by these jobs is considered the +training data. You will need some amount of user history in your recs query. It is typical to use the most recent user history +but need not be exactly what is in the training set, which may include a greater volume of historical data. Keeping the user +history for query purposes could be done with a database by storing it in a users table. In the example above the two +collaborative filtering actions are "purchase" and "view", but let's also add tags (taken from catalog categories or other +descriptive metadata). -We will need to create 1 cooccurrence indicator from the primary action (purchase) 1 cross-action cooccurrence indicator -from the secondary action (view) +We will need to create 1 cooccurrence indicator from the primary action (purchase) 1 cross-action cooccurrence indicator +from the secondary action (view) and 1 content indicator (tags). We'll have to run *spark-itemsimilarity* once and *spark-rowsimilarity* once. -We have described how to create the collaborative filtering indicator and cross-cooccurrence indicator for purchase and view (the [How to use Multiple User -Actions](#multiple-actions) section) but tags will be a slightly different process. We want to use the fact that -certain items have tags similar to the ones associated with a user's purchases. This is not a collaborative filtering indicator -but rather a "content" or "metadata" type indicator since you are not using other users' history, only the -individual that you are making recs for. This means that this method will make recommendations for items that have +We have described how to create the collaborative filtering indicator and cross-cooccurrence indicator for purchase and view (the [How to use Multiple User +Actions](#multiple-actions) section) but tags will be a slightly different process. We want to use the fact that +certain items have tags similar to the ones associated with a user's purchases. This is not a collaborative filtering indicator +but rather a "content" or "metadata" type indicator since you are not using other users' history, only the +individual that you are making recs for. This means that this method will make recommendations for items that have no collaborative filtering data, as happens with new items in a catalog. New items may have tags assigned but no one has purchased or viewed them yet. In the final query we will mix all 3 indicators. ##Content Indicator -To create a content-indicator we'll make use of the fact that the user has purchased items with certain tags. We want to find -items with the most similar tags. Notice that other users' behavior is not considered--only other item's tags. This defines a -content or metadata indicator. They are used when you want to find items that are similar to other items by using their +To create a content-indicator we'll make use of the fact that the user has purchased items with certain tags. We want to find +items with the most similar tags. Notice that other users' behavior is not considered--only other item's tags. This defines a +content or metadata indicator. They are used when you want to find items that are similar to other items by using their content or metadata, not by which users interacted with them. For this we need input of the form: @@ -387,9 +387,9 @@ The full collection will look like the tags column from a catalog DB. For our ec 9446577dwomen tops chambray clothing casual ... -We'll use *spark-rowimilairity* because we are looking for similar rows, which encode items in this case. As with the -collaborative filtering indicator and cross-cooccurrence indicator we use the --omitStrength option. The strengths created are -probabilistic log-likelihood ratios and so are used to filter unimportant similarities. Once the filtering or downsampling +We'll use *spark-rowimilairity* because we are looking for similar rows, which encode items in this case. As with the +collaborative filtering indicator and cross-cooccurrence indicator we use the --omitStrength option. The strengths created are +probabilistic log-likelihood ratios and so are used to filter unimportant similarities. Once the filtering or downsampling is finished we no longer need the strengths. We will get an indicator matrix of the form: itemIDlist-of-item IDs @@ -399,31 +399,31 @@ This is a content indicator since it has found other items with similar content 3459860b3459860b 3459860b 6749860c 5959860a 3434860a 3477860a 9446577d9446577d 9496577d 0943577d 8346577d 9442277d 9446577e - ... - + ... + We now have three indicators, two collaborative filtering type and one content type. ##Unified Recommender Query -The actual form of the query for recommendations will vary depending on your search engine but the intent is the same. -For a given user, map their history of an action or content to the correct indicator field and perform an OR'd query. +The actual form of the query for recommendations will vary depending on your search engine but the intent is the same. +For a given user, map their history of an action or content to the correct indicator field and perform an OR'd query. -We have 3 indicators, these are indexed by the search engine into 3 fields, we'll call them "purchase", "view", and "tags". +We have 3 indicators, these are indexed by the search engine into 3 fields, we'll call them "purchase", "view", and "tags". We take the user's history that corresponds to each indicator and create a query of the form: Query: field: purchase; q:user's-purchase-history field: view; q:user's view-history field: tags; q:user's-tags-associated-with-purchases - -The query will result in an ordered list of items recommended for purchase but skewed towards items with similar tags to -the ones the user has already purchased. -This is only an example and not necessarily the optimal way to create recs. It illustrates how business decisions can be -translated into recommendations. This technique can be used to skew recommendations towards intrinsic indicators also. -For instance you may want to put personalized popular item recs in a special place in the UI. Create a popularity indicator +The query will result in an ordered list of items recommended for purchase but skewed towards items with similar tags to +the ones the user has already purchased. + +This is only an example and not necessarily the optimal way to create recs. It illustrates how business decisions can be +translated into recommendations. This technique can be used to skew recommendations towards intrinsic indicators also. +For instance you may want to put personalized popular item recs in a special place in the UI. Create a popularity indicator by tagging items with some category of popularity (hot, warm, cold for instance) then -index that as a new indicator field and include the corresponding value in a query +index that as a new indicator field and include the corresponding value in a query on the popularity field. If we use the ecom example but use the query to get "hot" recommendations it might look like this: Query: diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/intro-itembased-hadoop.md b/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/intro-itembased-hadoop.md index 64ae6284bc..88d5070cb0 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/intro-itembased-hadoop.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/intro-itembased-hadoop.md @@ -3,7 +3,7 @@ layout: deprecated-docs title: (Deprecated) Item Based Reccommenders in Hadoop redirect_from: - /documentation/tutorials/map-reduce/recommender/intro-itembased-hadoop - - /documentation/tutorials/map-reduce/recommender/intro-itembased-hadoop.html + - /documentation/tutorials/map-reduce/recommender/intro-itembased-hadoop.html --- @@ -55,4 +55,4 @@ Running the command will execute a series of jobs the final product of which wil **Step 4: Making use of the output and doing more with Mahout** -The output file generated in our simple example can be transformed using your tool of choice and consumed by downstream applications. There exist a variety of configuration options for Mahout’s item based recommender to accommodate custom business requirements; exploring and testing various configurations to suit your needs will doubtless lead to additional questions. Our user community is accessible via our [mailing list](https://mahout.apache.org/community/mailing-lists.html) and the book *Mahout In Action* is a fantastic (but slightly outdated) starting point. +The output file generated in our simple example can be transformed using your tool of choice and consumed by downstream applications. There exist a variety of configuration options for Mahout’s item based recommender to accommodate custom business requirements; exploring and testing various configurations to suit your needs will doubtless lead to additional questions. Our user community is accessible via our [mailing list](https://mahout.apache.org/community/mailing-lists.html) and the book *Mahout In Action* is a fantastic (but slightly outdated) starting point. diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/matrix-factorization.md b/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/matrix-factorization.md index d64fad7773..6fb512bdc6 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/matrix-factorization.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/matrix-factorization.md @@ -11,14 +11,14 @@ redirect_from: # Introduction to Matrix Factorization for Recommendation Mining -In the mathematical discipline of linear algebra, a matrix decomposition -or matrix factorization is a dimensionality reduction technique that factorizes a matrix into a product of matrices, usually two. +In the mathematical discipline of linear algebra, a matrix decomposition +or matrix factorization is a dimensionality reduction technique that factorizes a matrix into a product of matrices, usually two. There are many different matrix decompositions, each finds use among a particular class of problems. In mahout, the SVDRecommender provides an interface to build recommender based on matrix factorization. The idea behind is to project the users and items onto a feature space and try to optimize U and M so that U \* (M^t) is as close to R as possible: - U is n * p user feature matrix, + U is n * p user feature matrix, M is m * p item feature matrix, M^t is the conjugate transpose of M, R is n * m rating matrix, n is the number of users, @@ -30,14 +30,14 @@ RMSE is defined as the squared root of the sum of squared errors at each known u So our matrix factorization target could be mathmatically defined as: find U and M, (U, M) = argmin(RMSE) = argmin(pow(SSE / K, 0.5)) - + SSE = sum(e(u,i)^2) e(u,i) = r(u, i) - U[u,] * (M[i,]^t) = r(u,i) - sum(U[u,f] * M[i,f]), f = 0, 1, .. p - 1 K is the number of known user item ratings. -Mahout has implemented matrix factorization based on +Mahout has implemented matrix factorization based on (1) SGD(Stochastic Gradient Descent) (2) ALSWR(Alternating-Least-Squares with Weighted-λ-Regularization). @@ -46,11 +46,11 @@ Mahout has implemented matrix factorization based on Stochastic gradient descent is a gradient descent optimization method for minimizing an objective function that is written as a su of differentiable functions. - Q(w) = sum(Q_i(w)), + Q(w) = sum(Q_i(w)), where w is the parameters to be estimated, Q(w) is the objective function that could be expressed as sum of differentiable functions, - Q_i(w) is associated with the i-th observation in the data set + Q_i(w) is associated with the i-th observation in the data set In practice, w is estimated using an iterative method at each single sample until an approximate miminum is obtained, @@ -69,9 +69,9 @@ where Q(u, i) is the objecive function for user u and item i, lambda is the regularization parameter to prevent overfitting. The algorithm is sketched as follows: - - init U and M with randomized value between 0.0 and 1.0 with standard Gaussian distribution - + + init U and M with randomized value between 0.0 and 1.0 with standard Gaussian distribution + for(iter = 0; iter < numIterations; iter++) { for(user u and item i with rating R[u,i]) @@ -82,11 +82,11 @@ The algorithm is sketched as follows: // p is the number of features for(f = 0; f < p; f++) { NU[u,f] = U[u,f] - alpha * d(Q(u,i))/d(U[u,f]) //optimize U[u,f] - = U[u, f] + alpha * (e(u,i) * M[i,f] - lambda * U[u,f]) + = U[u, f] + alpha * (e(u,i) * M[i,f] - lambda * U[u,f]) } for(f = 0; f < p; f++) { - M[i,f] = M[i,f] - alpha * d(Q(u,i))/d(M[i,f]) //optimize M[i,f] - = M[i,f] + alpha * (e(u,i) * U[u,f] - lambda * M[i,f]) + M[i,f] = M[i,f] - alpha * d(Q(u,i))/d(M[i,f]) //optimize M[i,f] + = M[i,f] + alpha * (e(u,i) * U[u,f] - lambda * M[i,f]) } U[u,] = NU[u,] } @@ -94,14 +94,14 @@ The algorithm is sketched as follows: ## SVD++ -SVD++ is an enhancement of the SGD matrix factorization. +SVD++ is an enhancement of the SGD matrix factorization. -It could be considered as an integration of latent factor model and neighborhood based model, considering not only how users rate, but also who has rated what. +It could be considered as an integration of latent factor model and neighborhood based model, considering not only how users rate, but also who has rated what. + +The complete model is a sum of 3 sub-models with complete prediction formula as follows: -The complete model is a sum of 3 sub-models with complete prediction formula as follows: - pr(u,i) = b[u,i] + fm + nm //user u and item i - + pr(u,i) is the predicted rating of user u on item i, b[u,i] = U + b(u) + b(i) fm = (q[i,]) * (p[u,] + pow(|N(u)|, -0.5) * sum(y[j,])), j is an item in N(u) @@ -111,7 +111,7 @@ The associated regularized squared error function to be minimized is: {sum((r[u,i] - pr[u,i]) * (r[u,i] - pr[u,i])) - lambda * (b(u) * b(u) + b(i) * b(i) + ||q[i,]||^2 + ||p[u,]||^2 + sum(||y[j,]||^2) + sum(w[i,j0] * w[i,j0]) + sum(c[i,j1] * c[i,j1]))} -b[u,i] is the baseline estimate of user u's predicted rating on item i. U is users' overall average rating and b(u) and b(i) indicate the observed deviations of user u and item i's ratings from average. +b[u,i] is the baseline estimate of user u's predicted rating on item i. U is users' overall average rating and b(u) and b(i) indicate the observed deviations of user u and item i's ratings from average. The baseline estimate is to adjust for the user and item effects - i.e, systematic tendencies for some users to give higher ratings than others and tendencies for some items to receive higher ratings than other items. @@ -119,29 +119,29 @@ for some items to receive higher ratings than other items. fm is the latent factor model to capture the interactions between user and item via a feature layer. q[i,] is the feature vector of item i, and the rest part of the formula represents user u with a user feature vector and a sum of features of items in N(u), N(u) is the set of items that user u have expressed preference, y[j,] is feature vector of an item in N(u). -nm is an extension of the classic item-based neighborhood model. -It captures not only the user's explicit ratings but also the user's implicit preferences. R(i;u;k) is the set of items that have got explicit rating from user u and only retain top k most similar items. r[u,j0] is the actual rating of user u on item j0, +nm is an extension of the classic item-based neighborhood model. +It captures not only the user's explicit ratings but also the user's implicit preferences. R(i;u;k) is the set of items that have got explicit rating from user u and only retain top k most similar items. r[u,j0] is the actual rating of user u on item j0, b[u,j0] is the corresponding baseline estimate. -The difference between r[u,j0] and b[u,j0] is weighted by a parameter w[i,j0], which could be thought as the similarity between item i and j0. +The difference between r[u,j0] and b[u,j0] is weighted by a parameter w[i,j0], which could be thought as the similarity between item i and j0. N[i;u;k] is the top k most similar items that have got the user's preference. -c[i;j1] is the paramter to be estimated. +c[i;j1] is the paramter to be estimated. -The value of w[i,j0] and c[i,j1] could be treated as the significance of the +The value of w[i,j0] and c[i,j1] could be treated as the significance of the user's explicit rating and implicit preference respectively. The parameters b, y, q, w, c are to be determined by minimizing the the associated regularized squared error function through gradient descent. We loop over all known ratings and for a given training case r[u,i], we apply gradient descent on the error function and modify the parameters by moving in the opposite direction of the gradient. For a complete analysis of the SVD++ algorithm, please refer to the paper [Yehuda Koren: Factorization Meets the Neighborhood: a Multifaceted Collaborative Filtering Model, KDD 2008](http://research.yahoo.com/files/kdd08koren.pdf). - -In Mahout,SVDPlusPlusFactorizer class is a simplified implementation of the SVD++ algorithm.It mainly uses the latent factor model with item feature vector, user feature vector and user's preference, with pr(u,i) = fm = (q[i,]) \* (p[u,] + pow(|N(u)|, -0.5) * sum(y[j,])) and the parameters to be determined are q, p, y. + +In Mahout,SVDPlusPlusFactorizer class is a simplified implementation of the SVD++ algorithm.It mainly uses the latent factor model with item feature vector, user feature vector and user's preference, with pr(u,i) = fm = (q[i,]) \* (p[u,] + pow(|N(u)|, -0.5) * sum(y[j,])) and the parameters to be determined are q, p, y. The update to q, p, y in each gradient descent step is: err(u,i) = r[u,i] - pr[u,i] - q[i,] = q[i,] + alpha * (err(u,i) * (p[u,] + pow(|N(u)|, -0.5) * sum(y[j,])) - lamda * q[i,]) + q[i,] = q[i,] + alpha * (err(u,i) * (p[u,] + pow(|N(u)|, -0.5) * sum(y[j,])) - lamda * q[i,]) p[u,] = p[u,] + alpha * (err(u,i) * q[i,] - lambda * p[u,]) for j that is an item in N(u): y[j,] = y[j,] + alpha * (err(u,i) * pow(|N(u)|, -0.5) * q[i,] - lambda * y[j,]) @@ -150,31 +150,31 @@ where alpha is the learning rate of gradient descent, N(u) is the items that use ## Parallel SGD -Mahout has a parallel SGD implementation in ParallelSGDFactorizer class. It shuffles the user ratings in every iteration and -generates splits on the shuffled ratings. Each split is handled by a thread to update the user features and item features using -vanilla SGD. +Mahout has a parallel SGD implementation in ParallelSGDFactorizer class. It shuffles the user ratings in every iteration and +generates splits on the shuffled ratings. Each split is handled by a thread to update the user features and item features using +vanilla SGD. -The implementation could be traced back to a lock-free version of SGD based on paper +The implementation could be traced back to a lock-free version of SGD based on paper [Hogwild!: A Lock-Free Approach to Parallelizing Stochastic Gradient Descent](http://www.eecs.berkeley.edu/~brecht/papers/hogwildTR.pdf). ## ALSWR -ALSWR is an iterative algorithm to solve the low rank factorization of user feature matrix U and item feature matrix M. +ALSWR is an iterative algorithm to solve the low rank factorization of user feature matrix U and item feature matrix M. The loss function to be minimized is formulated as the sum of squared errors plus [Tikhonov regularization](http://en.wikipedia.org/wiki/Tikhonov_regularization): L(R, U, M) = sum(pow((R[u,i] - U[u,]* (M[i,]^t)), 2)) + lambda * (sum(n(u) * ||U[u,]||^2) + sum(n(i) * ||M[i,]||^2)) - -At the beginning of the algorithm, M is initialized with the average item ratings as its first row and random numbers for the rest row. -In every iteration, we fix M and solve U by minimization of the cost function L(R, U, M), then we fix U and solve M by the minimization of +At the beginning of the algorithm, M is initialized with the average item ratings as its first row and random numbers for the rest row. + +In every iteration, we fix M and solve U by minimization of the cost function L(R, U, M), then we fix U and solve M by the minimization of the cost function similarly. The iteration stops until a certain stopping criteria is met. -To solve the matrix U when M is given, each user's feature vector is calculated by resolving a regularized linear least square error function +To solve the matrix U when M is given, each user's feature vector is calculated by resolving a regularized linear least square error function using the items the user has rated and their feature vectors: - 1/2 * d(L(R,U,M)) / d(U[u,f]) = 0 + 1/2 * d(L(R,U,M)) / d(U[u,f]) = 0 -Similary, when M is updated, we resolve a regularized linear least square error function using feature vectors of the users that have rated the +Similary, when M is updated, we resolve a regularized linear least square error function using feature vectors of the users that have rated the item and their feature vectors: 1/2 * d(L(R,U,M)) / d(M[i,f]) = 0 @@ -186,6 +186,5 @@ Mahout also offers a [parallel map-reduce implementation](https://mahout.apache. # Reference: [Stochastic gradient descent](http://en.wikipedia.org/wiki/Stochastic_gradient_descent) - -[ALSWR](http://www.hpl.hp.com/personal/Robert_Schreiber/papers/2008%20AAIM%20Netflix/netflix_aaim08%28submitted%29.pdf) +[ALSWR](http://www.hpl.hp.com/personal/Robert_Schreiber/papers/2008%20AAIM%20Netflix/netflix_aaim08%28submitted%29.pdf) diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/quickstart.md b/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/quickstart.md index 39de37ad85..33949d3f9c 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/quickstart.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/quickstart.md @@ -15,7 +15,7 @@ Recommenders have changed over the years. Mahout contains a long list of them, w To integrate with your application you will collect user interactions storing them in a DB and also in a from usable by Mahout. The simplest way to do this is to log user interactions to csv files (user-id, item-id). The DB should be setup to contain the last n user interactions, which will form part of the query for recommendations. -Mahout's spark-itemsimilarity will create a table of (item-id, list-of-similar-items) in csv form. Think of this as an item collection with one field containing the item-ids of similar items. Index this with your search engine. +Mahout's spark-itemsimilarity will create a table of (item-id, list-of-similar-items) in csv form. Think of this as an item collection with one field containing the item-ids of similar items. Index this with your search engine. When your application needs recommendations for a specific person, get the latest user history of interactions from the DB and query the indicator collection with this history. You will get back an ordered list of item-ids. These are your recommendations. You may wish to filter out any that the user has already seen but that will depend on your use case. diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/recommender-documentation.md b/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/recommender-documentation.md index 036453a406..d2fa603eaf 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/recommender-documentation.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/recommender-documentation.md @@ -41,7 +41,7 @@ Top-level packages define the Mahout interfaces to these key abstractions: Subpackages of *org.apache.mahout.cf.taste.impl* hold implementations of these interfaces. These are the pieces from which you will build your own -recommendation engine. That's it! +recommendation engine. That's it! ## Architecture @@ -126,23 +126,23 @@ Now we create a **UserNeighborhood** algorithm. Here we use nearest-3: UserNeighborhood neighborhood = new NearestNUserNeighborhood(3, userSimilarity, model);{code} - + Now we can create our **Recommender**, and add a caching decorator: - + Recommender recommender = new GenericUserBasedRecommender(model, neighborhood, userSimilarity); Recommender cachingRecommender = new CachingRecommender(recommender); - + Now we can get 10 recommendations for user ID "1234" — done! List recommendations = cachingRecommender.recommend(1234, 10); - + ## Item-based Recommender - + We could have created an item-based recommender instead. Item-based recommenders base recommendation not on user similarity, but on item similarity. In theory these are about the same approach to the problem, @@ -151,19 +151,19 @@ relatively fixed, more so than the similarity of two users. So, item-based recommenders can use pre-computed similarity values in the computations, which make them much faster. For large data sets, item-based recommenders are more appropriate. - + Let's start over, again with a **FileDataModel** to start: - + DataModel model = new FileDataModel(new File("data.txt")); - + We'll also need an **ItemSimilarity**. We could use **PearsonCorrelationSimilarity**, which computes item similarity in realtime, but, this is generally too slow to be useful. Instead, in a real application, you would feed a list of pre-computed correlations to a -**GenericItemSimilarity**: - +**GenericItemSimilarity**: + // Construct the list of pre-computed correlations Collection correlations = @@ -172,9 +172,9 @@ application, you would feed a list of pre-computed correlations to a new GenericItemSimilarity(correlations); - + Then we can finish as before to produce recommendations: - + Recommender recommender = new GenericItemBasedRecommender(model, itemSimilarity); @@ -222,7 +222,7 @@ code and third-party code you use doesn't call this method. Also consider the following tips: * Use **CachingRecommender** on top of your custom **Recommender** implementation. -* When using **JDBCDataModel**, make sure you wrap it with the **ReloadFromJDBCDataModel** to load data into memory!. +* When using **JDBCDataModel**, make sure you wrap it with the **ReloadFromJDBCDataModel** to load data into memory!. ### Algorithm Performance: Which One Is Best? diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/recommender-first-timer-faq.md b/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/recommender-first-timer-faq.md index 728b12758f..2e7a9f568e 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/recommender-first-timer-faq.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/recommender-first-timer-faq.md @@ -18,7 +18,7 @@ For the interested, these topics are treated in detail in the book [Mahout in Ac Don't start with a distributed, Hadoop-based recommender; take on that complexity only if necessary. Start with non-distributed recommenders. It -is simpler, has fewer requirements, and is more flexible. +is simpler, has fewer requirements, and is more flexible. As a crude rule of thumb, a system with up to 100M user-item associations (ratings, preferences) should "fit" onto one modern server machine with 4GB @@ -53,4 +53,4 @@ top a ReloadFromJDBCDataModel. This should give a reasonable starter system which responds fast. The nature of the system is that new data comes in from the file or database -only periodically -- perhaps on the order of minutes. +only periodically -- perhaps on the order of minutes. diff --git a/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/userbased-5-minutes.md b/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/userbased-5-minutes.md index 955b0989bc..554c3e751f 100644 --- a/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/userbased-5-minutes.md +++ b/website/Map_Reduce_Folder/tutorials/map-reduce/recommender/userbased-5-minutes.md @@ -19,7 +19,7 @@ Create a java project in your favorite IDE and make sure mahout is on the classp Mahout's recommenders expect interactions between users and items as input. The easiest way to supply such data to Mahout is in the form of a textfile, where every line has the format *userID,itemID,value*. Here *userID* and *itemID* refer to a particular user and a particular item, and *value* denotes the strength of the interaction (e.g. the rating given to a movie). -In this example, we'll use some made up data for simplicity. Create a file called "dataset.csv" and copy the following example interactions into the file. +In this example, we'll use some made up data for simplicity. Create a file called "dataset.csv" and copy the following example interactions into the file.
 1,10,1.0
@@ -81,9 +81,9 @@ Now we have all the pieces to create our recommender:
 
 UserBasedRecommender recommender = new GenericUserBasedRecommender(model, neighborhood, similarity);
 
- + We can easily ask the recommender for recommendations now. If we wanted to get three items recommended for the user with *userID* 2, we would do it like this: - +
 List recommendations = recommender.recommend(2, 3);
@@ -112,7 +112,7 @@ UserNeighborhood neighborhood = new ThresholdUserNeighborhood(0.1, similarity, d
 return new GenericUserBasedRecommender(dataModel, neighborhood, similarity);
 
-Now we have to create the code for the test. We'll check how much the recommender misses the real interaction strength on average. We employ an *AverageAbsoluteDifferenceRecommenderEvaluator* for this. The following code shows how to put the pieces together and run a hold-out test: +Now we have to create the code for the test. We'll check how much the recommender misses the real interaction strength on average. We employ an *AverageAbsoluteDifferenceRecommenderEvaluator* for this. The following code shows how to put the pieces together and run a hold-out test:
 DataModel model = new FileDataModel(new File("/path/to/dataset.csv"));
@@ -122,15 +122,4 @@ double result = evaluator.evaluate(builder, null, model, 0.9, 1.0);
 System.out.println(result);
 
-Note: if you run this test multiple times, you will get different results, because the splitting into trainingset and testset is done randomly. - - - - - - - - - - - +Note: if you run this test multiple times, you will get different results, because the splitting into trainingset and testset is done randomly. diff --git a/website/Map_Reduce_Folder/tutorials/misc/contributing-algos/index.md b/website/Map_Reduce_Folder/tutorials/misc/contributing-algos/index.md index e236a05e54..055ddf9641 100644 --- a/website/Map_Reduce_Folder/tutorials/misc/contributing-algos/index.md +++ b/website/Map_Reduce_Folder/tutorials/misc/contributing-algos/index.md @@ -6,10 +6,10 @@ redirect_from: - /documentation/tutorials/misc/contributing-algos/index.html --- -The Mahout community is driven by user contribution. If you have implemented an algorithm and are interested in +The Mahout community is driven by user contribution. If you have implemented an algorithm and are interested in sharing it with the rest of the community, we highly encourage you to contribute it to the codebase. However to -keep things from getting too out of control, we have instituted a standard API for our algorithms, and we ask you -contribute your algorithm in a way that conforms to this API as much as possible. You can always reach out on +keep things from getting too out of control, we have instituted a standard API for our algorithms, and we ask you +contribute your algorithm in a way that conforms to this API as much as possible. You can always reach out on [dev@mahout.apache.org](dev@mahout.apache.org) if you need help. In this example, let's say you've created a totally new algorithm- a regression algorithm called `Foo` @@ -19,21 +19,21 @@ illustrating how an algorithm would be added. ## Step 1: Create JIRA ticket -Go to [the Mahout JIRA board](https://issues.apache.org/jira/browse/MAHOUT/) and click the red "Create" button. +Go to [the Mahout JIRA board](https://issues.apache.org/jira/browse/MAHOUT/) and click the red "Create" button. If you don't see the button, you may need to be granted contributor rights on the JIRA board. Send a message to [dev@mahout.apache.org](dev@mahout.apache.org) and someone will add you. - -Once you click "Create" a dialog box will pop up. - + +Once you click "Create" a dialog box will pop up. + In the Summary box write a short description, something like `Implement Foo Algorithm` Under assignee- click "Assign to Me", this lets everyone know you're working on the algorithm. - + In the description, it would be good to describe the algorithm, and if possible link to a Wikipedia article about the algorithm/method or better yet an academic journal article. ![Jira Dialogue box](new-jira.png) -Once you've creted- a JIRA number will be assigned. For example, here is the JIRA ticket for me writing this tutorial. +Once you've creted- a JIRA number will be assigned. For example, here is the JIRA ticket for me writing this tutorial. ![New Issue Created](jira.png) @@ -42,12 +42,12 @@ This is MAHOUT-1980. Whatever number your new issue is assigned, we'll refer to ## Step 2. Clone Mahout and create branch Supposing you don't already have a copy of Mahout on your computer open a terminal and type the following - + git clone http://github.com/apache/mahout - + This will clone the Mahout source code into a directory called `mahout`. Go into that directory and create a new branch called `mahout-xxxx` (where `xxxx` is your JIRA number from step 1) - + cd mahout git checkout -b mahout-xxxx @@ -59,7 +59,7 @@ you use one, but you can do what you like. As far as screen shots go though, th Create a file `mahout/math-scala/src/main/scala/org/apache/mahout/math/algorithms/regression/FooModel.scala` The first thing to add to the file is a license: - + /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -78,30 +78,30 @@ The first thing to add to the file is a license: * specific language governing permissions and limitations * under the License. */ - -The next thing to add to the file is the `package` statement. + +The next thing to add to the file is the `package` statement. package org.apache.mahout.math.algorithms.regression - + And finally declare the Fitter and Model classes class Foo extends RegressorFitter { - + } - + class FooModel extends RegressorModel { - + } - -The Fitter class holds the methods for fitting which returns a Model, the Model class hold the parameters for the model and + +The Fitter class holds the methods for fitting which returns a Model, the Model class hold the parameters for the model and methods for using on new data sets. In a RegressorModel that is going to be a `predict()` method. - -In your algorithm, most of your code is going to go into the `.fit` method. Since this is just a silly example, we're + +In your algorithm, most of your code is going to go into the `.fit` method. Since this is just a silly example, we're don't really have anything to fit- so we're just going to return a FooModel (because that is what the Fitter must do) - + class Foo[K] extends RegressorFitter[K] { - + def fit(drmX : DrmLike[K], drmTarget: DrmLike[K], hyperparameters: (Symbol, Any)*): FooModel[K] ={ @@ -114,9 +114,9 @@ In your algorithm, most of your code is going to go into the `.fit` method. Sinc model } } - + class FooModel[K] extends RegressorModel[K] { - + def predict(drmPredictors: DrmLike[K]): DrmLike[K] = { drmPredictors.mapBlock(1) { case (keys, block: Matrix) => { @@ -133,23 +133,23 @@ as a reminder to you, the person writing a useful algorithm, that this is a good At this point it would be reasonable to try building Mahout and checking that your algorithm is working the way you expect it to mvn clean package -DskipTests - + I like to use the Mahout Spark-Shell for cases like this. cd $MAHOUT_HOME/bin ./mahout spark-shell - + Then once I'm in the shell: import org.apache.mahout.math.algorithms.regression.Foo - + val drmA = drmParallelize(dense((1.0, 1.2, 1.3, 1.4), (1.1, 1.5, 2.5, 1.0), (6.0, 5.2, -5.2, 5.3), (7.0,6.0, 5.0, 5.0), (10.0, 1.0, 20.0, -10.0))) - + val model = new Foo().fit(drmA(::, 0 until 2), drmA(::, 2 until 3)) - + model.predict(drmA).collect - -And everything seems to be in order. + +And everything seems to be in order. res5: org.apache.mahout.math.Matrix = { @@ -159,26 +159,26 @@ And everything seems to be in order. 3 => {0:1.0} 4 => {0:1.0} } - + ## Step 4. Working with Hyper Parameters -It's entirely likely you'll need to have hyper-parameters to tune your algorithm. +It's entirely likely you'll need to have hyper-parameters to tune your algorithm. -In Mahout we handle these with a map of Symbols. You might have noticed in the `fit` and `predict` methods we included +In Mahout we handle these with a map of Symbols. You might have noticed in the `fit` and `predict` methods we included `hyperparameters: (Symbol, Any)*` Well let's look at how we would work with those. -Suppose instead of always guessing "1.0" we wanted to guess some user defined number (a very silly algorithm). +Suppose instead of always guessing "1.0" we wanted to guess some user defined number (a very silly algorithm). -We'll be adding a parameter called `guessThisNumber` to the Fitter method. By convention, we usually create a function in +We'll be adding a parameter called `guessThisNumber` to the Fitter method. By convention, we usually create a function in the fitter called `setStandardHyperparameters` and let that take care of setting up all of our hyperparameters and then call -that function inside of fit. This keeps things nice and clean. +that function inside of fit. This keeps things nice and clean. class Foo[K] extends RegressorFitter[K] { - + var guessThisNumber: Double = _ - + def setStandardHyperparameters(hyperparameters: Map[Symbol, Any] = Map('foo -> None)): Unit = { guessThisNumber = hyperparameters.asInstanceOf[Map[Symbol, Double]].getOrElse('guessThisNumber, 1.0) } @@ -188,34 +188,34 @@ that function inside of fit. This keeps things nice and clean. /** * Normally one would have a lot more code here. */ - + var model = new FooModel[K] - + setStandardHyperparameters(hyperparameters.toMap) model.guessThisNumber = guessThisNumber model.summary = s"This model will always guess ${model.guessThisNumber}" model } } - + Also notice we set the _default value_ to 1.0. We also now have something to add into the summary string. To implement this, we'll need to broadcast the guessed number in the `predict` method. In Mahout you can only broadcast -`Vectors` and `Matrices`. We use `drmBroadcast` It might be tempting to use the broadcast method of the underlying engine, but -this is a no-no. The reason for this is that we want to keep our algorithm abstracted over multiple distributed engines. +`Vectors` and `Matrices`. We use `drmBroadcast` It might be tempting to use the broadcast method of the underlying engine, but +this is a no-no. The reason for this is that we want to keep our algorithm abstracted over multiple distributed engines. class FooModel[K] extends RegressorModel[K] { - + var guessThisNumber: Double = _ - + def predict(drmPredictors: DrmLike[K]): DrmLike[K] = { - + // This is needed for MapBlock implicit val ktag = drmPredictors.keyClassTag - + // This is needed for broadcasting implicit val ctx = drmPredictors.context - + val bcGuess = drmBroadcast(dvec(guessThisNumber)) drmPredictors.mapBlock(1) { case (keys, block: Matrix) => { @@ -225,22 +225,22 @@ this is a no-no. The reason for this is that we want to keep our algorithm abst } } } - + We can get pretty creative with what sort of information we can send out in the broadcast even when it's just Vectors and Matrices Here we get the single number we need by broadcasting it in a length 1 vector. We then `get` our number from that position. keys -> (outputBlock += bcGuess.get(0)) - + Let's open up the `$MAHOUT_HOME/bin/mahout spark-shell` and try out the hyperparameter import org.apache.mahout.math.algorithms.regression.Foo - + val drmA = drmParallelize(dense((1.0, 1.2, 1.3, 1.4), (1.1, 1.5, 2.5, 1.0), (6.0, 5.2, -5.2, 5.3), (7.0,6.0, 5.0, 5.0), (10.0, 1.0, 20.0, -10.0))) - + val model = new Foo().fit(drmA(::, 0 until 2), drmA(::, 2 until 3), 'guessThisNumber -> 2.0) - + model.predict(drmA).collect ## Step 5. Unit Tests @@ -253,71 +253,71 @@ of the algorithm- and then verify your algorithm generates the same results, or Since no one has implemented Foo in any other packages however, we will have to be very sure our algorithm correct by doing it by hand in R, and in a way others can verify. Actually- I'm not making a prototype of the Foo algorithm, but I do want to impress -that it _is_ very important that you do, so that the person reviewing the PR can verify your results. (Maybe there is an example +that it _is_ very important that you do, so that the person reviewing the PR can verify your results. (Maybe there is an example in the paper you read). -Since this is a regression model, open up +Since this is a regression model, open up $MAHOUT_HOME/math-scala/src/test/scala/org/apache/mahout/math/algorithms/RegressionSuiteBase.scala - -You'll see some other tests in there to get you started. You'll also see some R-Prototypes, and in the case of `cochrane orcutt` -where the R implementation had divergent results and my argument for why our algorithm was right. + +You'll see some other tests in there to get you started. You'll also see some R-Prototypes, and in the case of `cochrane orcutt` +where the R implementation had divergent results and my argument for why our algorithm was right. I'm going to create a new test called `foo test` and I'm going to build it similar to the example I've been using. test("foo") { import org.apache.mahout.math.algorithms.regression.Foo - + val drmA = drmParallelize(dense((1.0, 1.2, 1.3, 1.4), (1.1, 1.5, 2.5, 1.0), (6.0, 5.2, -5.2, 5.3), (7.0,6.0, 5.0, 5.0), (10.0, 1.0, 20.0, -10.0))) - + val model = new Foo().fit(drmA(::, 0 until 2), drmA(::, 2 until 3), 'guessThisNumber -> 2.0) - + val myAnswer = model.predict(drmA).collect val correctAnswer = dense( (2.0), (2.0), (2.0), (2.0), (2.0)) - - + + val epsilon = 1E-6 (myAnswer - correctAnswer).sum should be < epsilon } - -Note the use of `epsilon`. The answer really _should be_ 0.0. But, especially for more complicated algorithms, we allow + +Note the use of `epsilon`. The answer really _should be_ 0.0. But, especially for more complicated algorithms, we allow for a touch of machine rounding error. Now build and check your tests by building without skipping the tests mvn clean package - -## Step 6. Add documentation to the website. + +## Step 6. Add documentation to the website. Now you've created this awesome algorithm- time to do a little marketing! Create the following file: $MAHOUT_HOME/website/docs/algorithms/regression/foo.md In that file create a blank Jekyll template: - + --- layout: doc-page title: Foo - - + + --- - + ### About - + Foo is a very famous and useful algorithm. Let me tell you lots about it... - + [A mark down link](https://en.wikipedia.org/wiki/Foobar) - + ### Parameters - +
@@ -332,26 +332,26 @@ In that file create a blank Jekyll template:
- + ### Example import org.apache.mahout.math.algorithms.regression.Foo - + val drmA = drmParallelize(dense((1.0, 1.2, 1.3, 1.4), (1.1, 1.5, 2.5, 1.0), (6.0, 5.2, -5.2, 5.3), (7.0,6.0, 5.0, 5.0), (10.0, 1.0, 20.0, -10.0))) - + val model = new Foo().fit(drmA(::, 0 until 2), drmA(::, 2 until 3), 'guessThisNumber -> 2.0) - + model.predict(drmA).collect -The firse few lines between the `---` is the header, this has the title and tells Jekyll what sort of page this is, it knows +The firse few lines between the `---` is the header, this has the title and tells Jekyll what sort of page this is, it knows elsewhere based on that, how to compile the page (add navbars, etc). -The _About_ section, is your chance to really dive into the algorithm and its implementation. More is more. If you didn't have an +The _About_ section, is your chance to really dive into the algorithm and its implementation. More is more. If you didn't have an R prototype in the unit tests (or have a divergent answer from R) this is a good place to really expand on that. - + The _Parameters_ section is a good reference for users to know that the hyperparameters are and what they do. -The _Example_ section is a quick little example to get users started. You may have noticed I used the same code I used in -the unit test. That's something I do often, but there is no reason you have to. If you want to come up with a more illustrative example +The _Example_ section is a quick little example to get users started. You may have noticed I used the same code I used in +the unit test. That's something I do often, but there is no reason you have to. If you want to come up with a more illustrative example (or several illustrative examples) that's encouraged. Add links to the nav-bars @@ -376,12 +376,12 @@ If you're feeling really froggy- you're also welcome to add a tutorial :) ## Step 7. Commit Changes, Push to Github, and Open a PR Open a terminal, return to the `mahout` top directory and type - + git status - -You'll see `Changes not staged for commit`. -Any file you touched will be listed there, but we only want to stage the files we were in. +You'll see `Changes not staged for commit`. + +Any file you touched will be listed there, but we only want to stage the files we were in. For this tutorial it was @@ -390,12 +390,12 @@ For this tutorial it was git add website/docs/algorithms/regression/foo.md git add website/docs/_includes/algo_navbar.html git add website/docs/_includes/navbar.html - + Now we _commit_ our changes. We add a message that starts with `MAHOUT-xxxx` where `xxxx` is the JIRA number your issue was assigned, then a descriptive title. git commit -m "MAHOUT-xxxx Implement Foo Algorithm" - + Finally, _push_ the changes to your local repository. The `-u` flag will create a new branch git push -u origin mahout-xxxx @@ -414,4 +414,4 @@ Towards the top, off to the right, you'll see a link to "Pull Request", click on That's it! Thanks so much for contributing to Apache Mahout, users like you are what keep this project moving forward! -I've included [Foo.scala](Foo.scala) and [RegressionSuiteBase.scala](RegressionSuiteBase.scala) for reference. +I've included [Foo.scala](Foo.scala) and [RegressionSuiteBase.scala](RegressionSuiteBase.scala) for reference. diff --git a/website/Map_Reduce_Folder/tutorials/misc/how-to-build-an-app.md b/website/Map_Reduce_Folder/tutorials/misc/how-to-build-an-app.md index 6164bd3960..39e2bfc321 100644 --- a/website/Map_Reduce_Folder/tutorials/misc/how-to-build-an-app.md +++ b/website/Map_Reduce_Folder/tutorials/misc/how-to-build-an-app.md @@ -30,15 +30,15 @@ Using Mahout as a library in an application will require a little Scala code. Sc object CooccurrenceDriver extends App { } - + This will look a little different than Java since ```App``` does delayed initialization, which causes the body to be executed when the App is launched, just as in Java you would create a main method. Before we can execute something on Spark we'll need to create a context. We could use raw Spark calls here but default values are setup for a Mahout context by using the Mahout helper function. - implicit val mc = mahoutSparkContext(masterUrl = "local", + implicit val mc = mahoutSparkContext(masterUrl = "local", appName = "CooccurrenceDriver") - + We need to read in three files containing different interaction types. The files will each be read into a Mahout IndexedDataset. This allows us to preserve application-specific user and item IDs throughout the calculations. For example, here is data/purchase.csv: @@ -51,12 +51,12 @@ For example, here is data/purchase.csv: u4,iphone u4,galaxy -Mahout has a helper function that reads the text delimited files SparkEngine.indexedDatasetDFSReadElements. The function reads single element tuples (user-id,item-id) in a distributed way to create the IndexedDataset. Distributed Row Matrices (DRM) and Vectors are important data types supplied by Mahout and IndexedDataset is like a very lightweight Dataframe in R, it wraps a DRM with HashBiMaps for row and column IDs. +Mahout has a helper function that reads the text delimited files SparkEngine.indexedDatasetDFSReadElements. The function reads single element tuples (user-id,item-id) in a distributed way to create the IndexedDataset. Distributed Row Matrices (DRM) and Vectors are important data types supplied by Mahout and IndexedDataset is like a very lightweight Dataframe in R, it wraps a DRM with HashBiMaps for row and column IDs. One important thing to note about this example is that we read in all datasets before we adjust the number of rows in them to match the total number of users in the data. This is so the math works out [(A'A, A'B, A'C)](http://mahout.apache.org/users/algorithms/intro-cooccurrence-spark.html) even if some users took one action but not another there must be the same number of rows in all matrices. /** - * Read files of element tuples and create IndexedDatasets one per action. These + * Read files of element tuples and create IndexedDatasets one per action. These * share a userID BiMap but have their own itemID BiMaps */ def readActions(actionInput: Array[(String, String)]): Array[(String, IndexedDataset)] = { @@ -64,7 +64,7 @@ One important thing to note about this example is that we read in all datasets b val userDictionary: BiMap[String, Int] = HashBiMap.create() - // The first action named in the sequence is the "primary" action and + // The first action named in the sequence is the "primary" action and // begins to fill up the user dictionary for ( actionDescription <- actionInput ) {// grab the path to actions val action: IndexedDataset = SparkEngine.indexedDatasetDFSReadElements( @@ -73,11 +73,11 @@ One important thing to note about this example is that we read in all datasets b existingRowIDs = userDictionary) userDictionary.putAll(action.rowIDs) // put the name in the tuple with the indexedDataset - actions = actions :+ (actionDescription._1, action) + actions = actions :+ (actionDescription._1, action) } - // After all actions are read in the userDictonary will contain every user seen, - // even if they may not have taken all actions . Now we adjust the row rank of + // After all actions are read in the userDictonary will contain every user seen, + // even if they may not have taken all actions . Now we adjust the row rank of // all IndxedDataset's to have this number of rows // Note: this is very important or the cooccurrence calc may fail val numUsers = userDictionary.size() // one more than the cardinality @@ -95,7 +95,7 @@ Now that we have the data read in we can perform the cooccurrence calculation. // actions.map creates an array of just the IndeedDatasets val indicatorMatrices = SimilarityAnalysis.cooccurrencesIDSs( - actions.map(a => a._2)) + actions.map(a => a._2)) All we need to do now is write the indicators. @@ -116,14 +116,14 @@ The ```writeIndicators``` method uses the default write function ```dfsWrite```. val indicatorDir = OutputPath + indicator._1 indicator._2.dfsWrite( indicatorDir, - // Schema tells the writer to omit LLR strengths + // Schema tells the writer to omit LLR strengths // and format for search engine indexing - IndexedDatasetWriteBooleanSchema) + IndexedDatasetWriteBooleanSchema) } } - -See the Github project for the full source. Now we create a build.sbt to build the example. + +See the Github project for the full source. Now we create a build.sbt to build the example. name := "cooccurrence-driver" @@ -164,7 +164,7 @@ Building the examples from project's root folder: This will automatically set up some launcher scripts for the driver. To run execute $ target/pack/bin/cooc - + The driver will execute in Spark standalone mode and put the data in /path/to/3-input-cooc/data/indicators/*indicator-type* ## Using a Debugger @@ -172,7 +172,7 @@ To build and run this example in a debugger like IntelliJ IDEA. Install from the Open IDEA and go to the menu File->New->Project from existing sources->SBT->/path/to/3-input-cooc. This will create an IDEA project from ```build.sbt``` in the root directory. -At this point you may create a "Debug Configuration" to run. In the menu choose Run->Edit Configurations. Under "Default" choose "Application". In the dialog hit the elipsis button "..." to the right of "Environment Variables" and fill in your versions of JAVA_HOME, SPARK_HOME, and MAHOUT_HOME. In configuration editor under "Use classpath from" choose root-3-input-cooc module. +At this point you may create a "Debug Configuration" to run. In the menu choose Run->Edit Configurations. Under "Default" choose "Application". In the dialog hit the elipsis button "..." to the right of "Environment Variables" and fill in your versions of JAVA_HOME, SPARK_HOME, and MAHOUT_HOME. In configuration editor under "Use classpath from" choose root-3-input-cooc module. ![image](http://mahout.apache.org/images/debug-config.png) @@ -191,18 +191,18 @@ To make the CooccurrenceDriver.scala into a script make the following changes: * You won't need the context, since it is created when the shell is launched, comment that line out. * Replace the logger.info lines with println -* Remove the package info since it's not needed, this will produce the file in ```path/to/3-input-cooc/bin/CooccurrenceDriver.mscala```. +* Remove the package info since it's not needed, this will produce the file in ```path/to/3-input-cooc/bin/CooccurrenceDriver.mscala```. Note the extension ```.mscala``` to indicate we are using Mahout's scala extensions for math, otherwise known as [Mahout-Samsara](http://mahout.apache.org/users/environment/out-of-core-reference.html) To run the code make sure the output does not exist already $ rm -r /path/to/3-input-cooc/data/indicators - + Launch the Mahout + Spark shell: $ mahout spark-shell - + You'll see the Mahout splash: MAHOUT_LOCAL is set, so we don't add HADOOP_CONF_DIR to classpath. @@ -213,14 +213,14 @@ You'll see the Mahout splash: | | | | | | (_| | | | | (_) | |_| | |_ |_| |_| |_|\__,_|_| |_|\___/ \__,_|\__| version 0.10.0 - + Using Scala version 2.10.4 (Java HotSpot(TM) 64-Bit Server VM, Java 1.7.0_72) Type in expressions to have them evaluated. Type :help for more information. 15/04/26 09:30:48 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable Created spark context.. Mahout distributed context is available as "implicit val sdc". - mahout> + mahout> To load the driver type: @@ -233,12 +233,12 @@ To load the driver type: import org.apache.mahout.sparkbindings._ import scala.collection.immutable.HashMap defined module CooccurrenceDriver - mahout> + mahout> To run the driver type: mahout> CooccurrenceDriver.main(args = Array("")) - + You'll get some stats printed: Total number of users for all actions = 5 @@ -254,5 +254,5 @@ You'll get some stats printed: Number of rows for matrix = 5 Number of columns for matrix = 7 Number of rows after resize = 5 - + If you look in ```path/to/3-input-cooc/data/indicators``` you should find folders containing the indicator matrices. diff --git a/website/Map_Reduce_Folder/tutorials/misc/mahout-in-zeppelin/index.md b/website/Map_Reduce_Folder/tutorials/misc/mahout-in-zeppelin/index.md index fe2308d8e0..7be2b84976 100644 --- a/website/Map_Reduce_Folder/tutorials/misc/mahout-in-zeppelin/index.md +++ b/website/Map_Reduce_Folder/tutorials/misc/mahout-in-zeppelin/index.md @@ -7,29 +7,29 @@ redirect_from: --- -** DEPRECATED : While this page is useful for learning how to set up Mahout in Zeppelin, we strongly reccomend using a +** DEPRECATED : While this page is useful for learning how to set up Mahout in Zeppelin, we strongly reccomend using a pre-built Docker container for trying out Mahout. See instructions [here](http://mahout.apache.org/docs/latest/tutorials/misc/getting-started-with-zeppelin/) ** The [Apache Zeppelin](http://zeppelin.apache.org) is an exciting notebooking tool, designed for working with Big Data -applications. It comes with great integration for graphing in R and Python, supports multiple langauges in a single -notebook (and facilitates sharing of variables between interpreters), and makes working with Spark and Flink in an interactive environment (either locally or in cluster mode) a +applications. It comes with great integration for graphing in R and Python, supports multiple langauges in a single +notebook (and facilitates sharing of variables between interpreters), and makes working with Spark and Flink in an interactive environment (either locally or in cluster mode) a breeze. Of course, it does lots of other cool things too- but those are the features we're going to take advantage of. ### Step1: Download and Install Zeppelin Zeppelin binaries by default use Spark 2.1 / Scala 2.11, until Mahout puts out Spark 2.1/Scala 2.11 binaries you have -two options. +two options. #### Option 1: Build Mahout for Spark 2.1/Scala 2.11 **Build Mahout** Follow the standard procedures for building Mahout, except manually set the Spark and Scala versions - the easiest way being: - + git clone http://github.com/apache/mahout cd mahout mvn clean package -Dspark.version=2.1.0 -Dscala.version=2.11.8 -Dscala.compat.version=2.11 -DskipTests - + **Download Zeppelin** @@ -54,12 +54,12 @@ We'll use Mahout binaries from Maven, so all you need to do is clone, and build After it builds successfully... bin/zeppelin-daemon.sh start - + And that's it. Open a web browser and surf to [http://localhost:8080](http://localhost:8080) ### Step2: Create the Mahout Spark Interpreter -After opening your web browser and surfing to [http://localhost:8080](http://localhost:8080), click on the `Anonymous` +After opening your web browser and surfing to [http://localhost:8080](http://localhost:8080), click on the `Anonymous` button on the top right corner, which will open a drop down. Then click `Interpreter`. ![Screen Shot1](zeppelin1.png) @@ -75,7 +75,7 @@ named it later in the tutorial) In the **Interpreter group** drop down, select `spark`. A bunch of other settings will now auto-populate. -Scroll to the bottom of the **Properties** list. In the last row, you'll see two blank boxes. +Scroll to the bottom of the **Properties** list. In the last row, you'll see two blank boxes. Add the following properies by clicking the "+" button to the right. @@ -100,11 +100,11 @@ Add the following properies by clicking the "+" button to the right. spark.kryoserializer.buffer.max 600m - + spark.serializer org.apache.spark.serializer.KryoSerializer - + @@ -132,7 +132,7 @@ Where `/path/to/mahout` is the path to the directory where you've built mahout. /path/to/mahout/mahout-spark_2.11-0.14.jar - + /path/to/mahout/mahout-spark_2.11-0.14.jar-dependeny-reduced.jar @@ -140,7 +140,7 @@ Where `/path/to/mahout` is the path to the directory where you've built mahout. -#### If you chose Option2 in Step 1: +#### If you chose Option2 in Step 1:
@@ -163,7 +163,7 @@ Where `/path/to/mahout` is the path to the directory where you've built mahout. - +
org.apache.mahout:mahout-native-viennacl-omp_2.11:0.14
@@ -182,99 +182,99 @@ _**OPTIONALLY**_ You can add **one** of the following artifacts for CPU/GPU acce org.apache.mahout:mahout-native-viennacl_2.11:0.14 ViennaCL GPU Accelerated - + org.apache.mahout:mahout-native-viennacl-omp_2.11:0.14 ViennaCL-OMP CPU Accelerated (use this if you don't have a good graphics card) - + -Make sure to click "Save" and you're all set. +Make sure to click "Save" and you're all set. ### Step 4. Rock and Roll. -Mahout in Zeppelin, unlike the Mahout Shell, won't take care of importing the Mahout libraries or creating the -`MahoutSparkContext`, we need to do that manually. This is easy though. When ever you start Zeppelin (or restart) the +Mahout in Zeppelin, unlike the Mahout Shell, won't take care of importing the Mahout libraries or creating the +`MahoutSparkContext`, we need to do that manually. This is easy though. When ever you start Zeppelin (or restart) the Mahout interpreter, you'll need to run the following code first: %sparkMahout - + import org.apache.mahout.math._ import org.apache.mahout.math.scalabindings._ import org.apache.mahout.math.drm._ import org.apache.mahout.math.scalabindings.RLikeOps._ import org.apache.mahout.math.drm.RLikeDrmOps._ import org.apache.mahout.sparkbindings._ - + implicit val sdc: org.apache.mahout.sparkbindings.SparkDistributedContext = sc2sdc(sc) - + At this point, you have a Zeppelin Interpreter which will behave like the `$MAHOUT_HOME/bin/mahout spark-shell` - + Except, much much more. At the begining I mentioned a few important features of Zeppelin, that we could leverage to use Zeppelin for visualizatoins. #### Example 1: Visualizing a Matrix (Sample) with R -In Mahout we can use `Matrices.symmetricUniformView` to create a Gaussian Matrix. +In Mahout we can use `Matrices.symmetricUniformView` to create a Gaussian Matrix. -We can use `.mapBlock` and some clever code to create a 3D Gausian Matrix. +We can use `.mapBlock` and some clever code to create a 3D Gausian Matrix. -We can use `.drmSampleToTsv` to take a sample of the matrix and turn it in to a tab seperated string. We take a sample of - the matrix because, since we are dealing with "big" data, we wouldn't want to try to collect and plot the entire matrix, +We can use `.drmSampleToTsv` to take a sample of the matrix and turn it in to a tab seperated string. We take a sample of + the matrix because, since we are dealing with "big" data, we wouldn't want to try to collect and plot the entire matrix, however, IF we knew we had a small matrix and we DID want to sample the entire thing, then we could sample `100.0` e.g. 100%. - -Finally we use `z.put(...)` to put a variable into Zeppelin's `ResourcePool` a block of memory shared by all interpreters. + +Finally we use `z.put(...)` to put a variable into Zeppelin's `ResourcePool` a block of memory shared by all interpreters. %sparkMahout - + val mxRnd3d = Matrices.symmetricUniformView(5000, 3, 1234) val drmRand3d = drmParallelize(mxRnd3d) - + val drmGauss = drmRand3d.mapBlock() {case (keys, block) => val blockB = block.like() for (i <- 0 until block.nrow) { val x: Double = block(i, 0) val y: Double = block(i, 1) val z: Double = block(i, 2) - + blockB(i, 0) = x blockB(i, 1) = y blockB(i, 2) = Math.exp(-((Math.pow(x, 2)) + (Math.pow(y, 2)))/2) } keys -> blockB } - + resourcePool.put("gaussDrm", drm.drmSampleToTSV(drmGauss, 50.0)) - + Here we sample 50% of the matrix and put it in the `ResourcePool` under a variable named "gaussDrm". Now, for the exciting part. Scala doesn't have a lot of great graphing utilities. But you know who does? R and Python. So -instead of trying to akwardly visualize our data using Scala, let's just use R and Python. +instead of trying to akwardly visualize our data using Scala, let's just use R and Python. We start the Spark R interpreter (we do this because the regular R interpreter doesn't have access to the resource pools). -We `z.get` the variable we just put in. +We `z.get` the variable we just put in. -We use R's `read.table` to read the string- this is very similar to how we would read a tsv file in R. +We use R's `read.table` to read the string- this is very similar to how we would read a tsv file in R. -Then we plot the data using the R `scatterplot3d` package. +Then we plot the data using the R `scatterplot3d` package. **Note** you may need to install `scatterplot3d`. In Ubuntu, do this with `sudo apt-get install r-cran-scatterplot3d` %spark.r {"imageWidth": "400px"} - + library(scatterplot3d) - - + + gaussStr = z.get("gaussDrm") data <- read.table(text= gaussStr, sep="\t", header=FALSE) - + scatterplot3d(data, color="green") -![A neat plot](zeppelin3.png) \ No newline at end of file +![A neat plot](zeppelin3.png) diff --git a/website/Map_Reduce_Folder/tutorials/samsara/classify-a-doc-from-the-shell.md b/website/Map_Reduce_Folder/tutorials/samsara/classify-a-doc-from-the-shell.md index 55e179cfe0..34b74955df 100644 --- a/website/Map_Reduce_Folder/tutorials/samsara/classify-a-doc-from-the-shell.md +++ b/website/Map_Reduce_Folder/tutorials/samsara/classify-a-doc-from-the-shell.md @@ -10,13 +10,13 @@ redirect_from: # Building a text classifier in Mahout's Spark Shell -This tutorial will take you through the steps used to train a Multinomial Naive Bayes model and create a text classifier based on that model using the ```mahout spark-shell```. +This tutorial will take you through the steps used to train a Multinomial Naive Bayes model and create a text classifier based on that model using the ```mahout spark-shell```. ## Prerequisites This tutorial assumes that you have your Spark environment variables set for the ```mahout spark-shell``` see: [Playing with Mahout's Shell](http://mahout.apache.org/users/sparkbindings/play-with-shell.html). As well we assume that Mahout is running in cluster mode (i.e. with the ```MAHOUT_LOCAL``` environment variable **unset**) as we'll be reading and writing to HDFS. ## Downloading and Vectorizing the Wikipedia dataset -*As of Mahout v. 0.10.0, we are still reliant on the MapReduce versions of ```mahout seqwiki``` and ```mahout seq2sparse``` to extract and vectorize our text. A* [*Spark implementation of seq2sparse*](https://issues.apache.org/jira/browse/MAHOUT-1663) *is in the works for Mahout v. 0.11.* However, to download the Wikipedia dataset, extract the bodies of the documentation, label each document and vectorize the text into TF-IDF vectors, we can simpmly run the [wikipedia-classifier.sh](https://github.com/apache/mahout/blob/master/examples/bin/classify-wikipedia.sh) example. +*As of Mahout v. 0.10.0, we are still reliant on the MapReduce versions of ```mahout seqwiki``` and ```mahout seq2sparse``` to extract and vectorize our text. A* [*Spark implementation of seq2sparse*](https://issues.apache.org/jira/browse/MAHOUT-1663) *is in the works for Mahout v. 0.11.* However, to download the Wikipedia dataset, extract the bodies of the documentation, label each document and vectorize the text into TF-IDF vectors, we can simpmly run the [wikipedia-classifier.sh](https://github.com/apache/mahout/blob/master/examples/bin/classify-wikipedia.sh) example. Please select a number to choose the corresponding task to run 1. CBayes (may require increased heap space on yarn) @@ -28,7 +28,7 @@ Enter (2). This will download a large recent XML dump of the Wikipedia database, ## Getting Started -Launch the ```mahout spark-shell```. There is an example script: ```spark-document-classifier.mscala``` (.mscala denotes a Mahout-Scala script which can be run similarly to an R script). We will be walking through this script for this tutorial but if you wanted to simply run the script, you could just issue the command: +Launch the ```mahout spark-shell```. There is an example script: ```spark-document-classifier.mscala``` (.mscala denotes a Mahout-Scala script which can be run similarly to an R script). We will be walking through this script for this tutorial but if you wanted to simply run the script, you could just issue the command: mahout> :load /path/to/mahout/examples/bin/spark-document-classifier.mscala @@ -63,11 +63,11 @@ Hadoop imports needed to read our dictionary: val model = SparkNaiveBayes.train(aggregatedObservations, labelIndex, false) val resAnalyzer = SparkNaiveBayes.test(model, fullData, false) println(resAnalyzer) - + printing the ```ResultAnalyzer``` will display the confusion matrix. ## Read in the dictionary and document frequency count from HDFS - + val dictionary = sdc.sequenceFile(pathToData + "wikipediaVecs/dictionary.file-0", classOf[Text], classOf[IntWritable]) @@ -76,16 +76,16 @@ printing the ```ResultAnalyzer``` will display the confusion matrix. classOf[LongWritable]) // setup the dictionary and document frequency count as maps - val dictionaryRDD = dictionary.map { + val dictionaryRDD = dictionary.map { case (wKey, wVal) => wKey.asInstanceOf[Text] - .toString() -> wVal.get() + .toString() -> wVal.get() } - + val documentFrequencyCountRDD = documentFrequencyCount.map { case (wKey, wVal) => wKey.asInstanceOf[IntWritable] - .get() -> wVal.get() + .get() -> wVal.get() } - + val dictionaryMap = dictionaryRDD.collect.map(x => x._1.toString -> x._2.toInt).toMap val dfCountMap = documentFrequencyCountRDD.collect.map(x => x._1.toInt -> x._2.toLong).toMap @@ -100,7 +100,7 @@ For this simple example, our function ```vectorizeDocument(...)``` will tokenize .toLowerCase .split(" ") .groupBy(identity) - .mapValues(_.length) + .mapValues(_.length) val vec = new RandomAccessSparseVector(dictionaryMap.size) val totalDFSize = dfMap(-1) val docSize = wordCounts.size @@ -126,17 +126,17 @@ For this simple example, our function ```vectorizeDocument(...)``` will tokenize val labelMap = model.labelIndex val numLabels = model.numLabels val reverseLabelMap = labelMap.map(x => x._2 -> x._1) - + // instantiate the correct type of classifier val classifier = model.isComplementary match { case true => new ComplementaryNBClassifier(model) case _ => new StandardNBClassifier(model) } -## Define an argmax function +## Define an argmax function The label with the highest score wins the classification for a given document. - + def argmax(v: Vector): (Int, Double) = { var bestIdx: Int = Integer.MIN_VALUE var bestScore: Double = Integer.MIN_VALUE.asInstanceOf[Int].toDouble @@ -158,7 +158,7 @@ The label with the highest score wins the classification for a given document. } ## Two sample news articles: United States Football and United Kingdom Football - + // A random United States football article // http://www.reuters.com/article/2015/01/28/us-nfl-superbowl-security-idUSKBN0L12JR20150128 val UStextToClassify = new String("(Reuters) - Super Bowl security officials acknowledge" + @@ -170,9 +170,9 @@ The label with the highest score wins the classification for a given document. " will battle. Deadly shootings in Paris and arrest of suspects in Belgium, Greece and" + " Germany heightened fears of more attacks around the world and social media accounts" + " linked to Middle East militant groups have carried a number of threats to attack" + - " high-profile U.S. events. There is no specific credible threat, said Johnson, who" + + " high-profile U.S. events. There is no specific credible threat, said Johnson, who" + " has appointed a federal coordination team to work with local, state and federal" + - " agencies to ensure safety of fans, players and other workers associated with the" + + " agencies to ensure safety of fans, players and other workers associated with the" + " Super Bowl. I'm confident we will have a safe and secure and successful event." + " Sunday's game has been given a Special Event Assessment Rating (SEAR) 1 rating, the" + " same as in previous years, except for the year after the Sept. 11, 2001 attacks, when" + @@ -192,7 +192,7 @@ The label with the highest score wins the classification for a given document. " every confidence the public safety agencies that represented in the planning process" + " are going to have their best and brightest out there this weekend and we will have" + " a very safe Super Bowl.") - + // A random United Kingdom football article // http://www.reuters.com/article/2015/01/26/manchester-united-swissquote-idUSL6N0V52RZ20150126 val UKtextToClassify = new String("(Reuters) - Manchester United have signed a sponsorship" + @@ -222,22 +222,22 @@ The label with the highest score wins the classification for a given document. " significant growth even within categories. United have endured a tricky transition" + " following the retirement of manager Alex Ferguson in 2013, finishing seventh in the" + " Premier League last season and missing out on a place in the lucrative Champions League." + - " ($1 = 0.8910 Swiss francs) (Writing by Neil Maidment, additional reporting by Jemima" + + " ($1 = 0.8910 Swiss francs) (Writing by Neil Maidment, additional reporting by Jemima" + " Kelly; editing by Keith Weir)") ## Vectorize and classify our documents val usVec = vectorizeDocument(UStextToClassify, dictionaryMap, dfCountMap) val ukVec = vectorizeDocument(UKtextToClassify, dictionaryMap, dfCountMap) - + println("Classifying the news article about superbowl security (united states)") classifyDocument(usVec) - + println("Classifying the news article about Manchester United (united kingdom)") classifyDocument(ukVec) -## Tie everything together in a new method to classify text - +## Tie everything together in a new method to classify text + def classifyText(txt: String): String = { val v = vectorizeDocument(txt, dictionaryMap, dfCountMap) classifyDocument(v) @@ -247,13 +247,13 @@ The label with the highest score wins the classification for a given document. classifyText("Hello world from Queens") classifyText("Hello world from London") - + ## Model persistance You can save the model to HDFS: model.dfsWrite("/path/to/model") - + And retrieve it with: val model = NBModel.dfsRead("/path/to/model") diff --git a/website/Map_Reduce_Folder/tutorials/samsara/play-with-shell.md b/website/Map_Reduce_Folder/tutorials/samsara/play-with-shell.md index ea0833e92a..ceeb510e6d 100644 --- a/website/Map_Reduce_Folder/tutorials/samsara/play-with-shell.md +++ b/website/Map_Reduce_Folder/tutorials/samsara/play-with-shell.md @@ -7,7 +7,7 @@ redirect_from: --- -# Playing with Mahout's Spark Shell +# Playing with Mahout's Spark Shell This tutorial will show you how to play with Mahout's scala DSL for linear algebra and its Spark shell. **Please keep in mind that this code is still in a very early experimental stage**. @@ -21,25 +21,25 @@ We'll use an excerpt of a publicly available [dataset about cereals](http://lib. Name | protein | fat | carbo | sugars | rating :-----------------------|:--------|:----|:------|:-------|:--------- Apple Cinnamon Cheerios | 2 | 2 | 10.5 | 10 | 29.509541 -Cap'n'Crunch | 1 | 2 | 12 | 12 | 18.042851 +Cap'n'Crunch | 1 | 2 | 12 | 12 | 18.042851 Cocoa Puffs | 1 | 1 | 12 | 13 | 22.736446 -Froot Loops | 2 | 1 | 11 | 13 | 32.207582 +Froot Loops | 2 | 1 | 11 | 13 | 32.207582 Honey Graham Ohs | 1 | 2 | 12 | 11 | 21.871292 -Wheaties Honey Gold | 2 | 1 | 16 | 8 | 36.187559 +Wheaties Honey Gold | 2 | 1 | 16 | 8 | 36.187559 Cheerios | 6 | 2 | 17 | 1 | 50.764999 Clusters | 3 | 2 | 13 | 7 | 40.400208 -Great Grains Pecan | 3 | 3 | 13 | 4 | 45.811716 +Great Grains Pecan | 3 | 3 | 13 | 4 | 45.811716 ## Installing Mahout & Spark on your local machine -We describe how to do a quick toy setup of Spark & Mahout on your local machine, so that you can run this example and play with the shell. +We describe how to do a quick toy setup of Spark & Mahout on your local machine, so that you can run this example and play with the shell. 1. Download [Apache Spark 1.6.2](http://d3kbcqa49mib13.cloudfront.net/spark-1.6.2-bin-hadoop2.6.tgz) and unpack the archive file 1. Change to the directory where you unpacked Spark and type ```sbt/sbt assembly``` to build it 1. Create a directory for Mahout somewhere on your machine, change to there and checkout the master branch of Apache Mahout from GitHub ```git clone https://github.com/apache/mahout mahout``` 1. Change to the ```mahout``` directory and build mahout using ```mvn -DskipTests clean install``` - + ## Starting Mahout's Spark shell 1. Goto the directory where you unpacked Spark and type ```sbin/start-all.sh``` to locally start Spark @@ -48,8 +48,8 @@ We describe how to do a quick toy setup of Spark & Mahout on your local machine, export SPARK_HOME=[directory where you unpacked Spark] export MASTER=[url of the Spark master]
- 1. Finally, change to the directory where you unpacked Mahout and type ```bin/mahout spark-shell```, -you should see the shell starting and get the prompt ```mahout> ```. Check + 1. Finally, change to the directory where you unpacked Mahout and type ```bin/mahout spark-shell```, +you should see the shell starting and get the prompt ```mahout> ```. Check [FAQ](http://mahout.apache.org/users/sparkbindings/faq.html) for further troubleshooting. ## Implementation @@ -74,15 +74,15 @@ val drmData = drmParallelize(dense( numPartitions = 2); -Have a look at this matrix. The first four columns represent the ingredients -(our features) and the last column (the rating) is the target variable for -our regression. [Linear regression](https://en.wikipedia.org/wiki/Linear_regression) -assumes that the **target variable** `\(\mathbf{y}\)` is generated by the -linear combination of **the feature matrix** `\(\mathbf{X}\)` with the +Have a look at this matrix. The first four columns represent the ingredients +(our features) and the last column (the rating) is the target variable for +our regression. [Linear regression](https://en.wikipedia.org/wiki/Linear_regression) +assumes that the **target variable** `\(\mathbf{y}\)` is generated by the +linear combination of **the feature matrix** `\(\mathbf{X}\)` with the **parameter vector** `\(\boldsymbol{\beta}\)` plus the - **noise** `\(\boldsymbol{\varepsilon}\)`, summarized in the formula -`\(\mathbf{y}=\mathbf{X}\boldsymbol{\beta}+\boldsymbol{\varepsilon}\)`. -Our goal is to find an estimate of the parameter vector + **noise** `\(\boldsymbol{\varepsilon}\)`, summarized in the formula +`\(\mathbf{y}=\mathbf{X}\boldsymbol{\beta}+\boldsymbol{\varepsilon}\)`. +Our goal is to find an estimate of the parameter vector `\(\boldsymbol{\beta}\)` that explains the data very well. As a first step, we extract `\(\mathbf{X}\)` and `\(\mathbf{y}\)` from our data matrix. We get *X* by slicing: we take all rows (denoted by ```::```) and the first four columns, which have the ingredients in milligrams as content. Note that the result is again a DRM. The shell will not execute this code yet, it saves the history of operations and defers the execution until we really access a result. **Mahout's DSL automatically optimizes and parallelizes all operations on DRMs and runs them on Apache Spark.** @@ -97,7 +97,7 @@ Next, we extract the target variable vector *y*, the fifth column of the data ma val y = drmData.collect(::, 4) -Now we are ready to think about a mathematical way to estimate the parameter vector *β*. A simple textbook approach is [ordinary least squares (OLS)](https://en.wikipedia.org/wiki/Ordinary_least_squares), which minimizes the sum of residual squares between the true target variable and the prediction of the target variable. In OLS, there is even a closed form expression for estimating `\(\boldsymbol{\beta}\)` as +Now we are ready to think about a mathematical way to estimate the parameter vector *β*. A simple textbook approach is [ordinary least squares (OLS)](https://en.wikipedia.org/wiki/Ordinary_least_squares), which minimizes the sum of residual squares between the true target variable and the prediction of the target variable. In OLS, there is even a closed form expression for estimating `\(\boldsymbol{\beta}\)` as `\(\left(\mathbf{X}^{\top}\mathbf{X}\right)^{-1}\mathbf{X}^{\top}\mathbf{y}\)`. The first thing which we compute for this is `\(\mathbf{X}^{\top}\mathbf{X}\)`. The code for doing this in Mahout's scala DSL maps directly to the mathematical formula. The operation ```.t()``` transposes a matrix and analogous to R ```%*%``` denotes matrix multiplication. @@ -111,12 +111,12 @@ The same is true for computing `\(\mathbf{X}^{\top}\mathbf{y}\)`. We can simply val drmXty = drmX.t %*% y -We're nearly done. The next step we take is to fetch `\(\mathbf{X}^{\top}\mathbf{X}\)` and -`\(\mathbf{X}^{\top}\mathbf{y}\)` into the memory of our driver machine (we are targeting -features matrices that are tall and skinny , -so we can assume that `\(\mathbf{X}^{\top}\mathbf{X}\)` is small enough -to fit in). Then, we provide them to an in-memory solver (Mahout provides -the an analog to R's ```solve()``` for that) which computes ```beta```, our +We're nearly done. The next step we take is to fetch `\(\mathbf{X}^{\top}\mathbf{X}\)` and +`\(\mathbf{X}^{\top}\mathbf{y}\)` into the memory of our driver machine (we are targeting +features matrices that are tall and skinny , +so we can assume that `\(\mathbf{X}^{\top}\mathbf{X}\)` is small enough +to fit in). Then, we provide them to an in-memory solver (Mahout provides +the an analog to R's ```solve()``` for that) which computes ```beta```, our OLS estimate of the parameter vector `\(\boldsymbol{\beta}\)`.
@@ -126,16 +126,16 @@ val Xty = drmXty.collect(::, 0)
 val beta = solve(XtX, Xty)
 
-That's it! We have a implemented a distributed linear regression algorithm -on Apache Spark. I hope you agree that we didn't have to worry a lot about -parallelization and distributed systems. The goal of Mahout's linear algebra -DSL is to abstract away the ugliness of programming a distributed system -as much as possible, while still retaining decent performance and +That's it! We have a implemented a distributed linear regression algorithm +on Apache Spark. I hope you agree that we didn't have to worry a lot about +parallelization and distributed systems. The goal of Mahout's linear algebra +DSL is to abstract away the ugliness of programming a distributed system +as much as possible, while still retaining decent performance and scalability. -We can now check how well our model fits its training data. -First, we multiply the feature matrix `\(\mathbf{X}\)` by our estimate of -`\(\boldsymbol{\beta}\)`. Then, we look at the difference (via L2-norm) of +We can now check how well our model fits its training data. +First, we multiply the feature matrix `\(\mathbf{X}\)` by our estimate of +`\(\boldsymbol{\beta}\)`. Then, we look at the difference (via L2-norm) of the target variable `\(\mathbf{y}\)` to the fitted target variable:
@@ -143,18 +143,18 @@ val yFitted = (drmX %*% beta).collect(::, 0)
 (y - yFitted).norm(2)
 
-We hope that we could show that Mahout's shell allows people to interactively and incrementally write algorithms. We have entered a lot of individual commands, one-by-one, until we got the desired results. We can now refactor a little by wrapping our statements into easy-to-use functions. The definition of functions follows standard scala syntax. +We hope that we could show that Mahout's shell allows people to interactively and incrementally write algorithms. We have entered a lot of individual commands, one-by-one, until we got the desired results. We can now refactor a little by wrapping our statements into easy-to-use functions. The definition of functions follows standard scala syntax. -We put all the commands for ordinary least squares into a function ```ols```. +We put all the commands for ordinary least squares into a function ```ols```.
-def ols(drmX: DrmLike[Int], y: Vector) = 
+def ols(drmX: DrmLike[Int], y: Vector) =
   solve(drmX.t %*% drmX, drmX.t %*% y)(::, 0)
 
 
Note that DSL declares implicit `collect` if coersion rules require an in-core argument. Hence, we can simply -skip explicit `collect`s. +skip explicit `collect`s. Next, we define a function ```goodnessOfFit``` that tells how well a model fits the target variable: @@ -165,11 +165,11 @@ def goodnessOfFit(drmX: DrmLike[Int], beta: Vector, y: Vector) = { } -So far we have left out an important aspect of a standard linear regression -model. Usually there is a constant bias term added to the model. Without -that, our model always crosses through the origin and we only learn the -right angle. An easy way to add such a bias term to our model is to add a -column of ones to the feature matrix `\(\mathbf{X}\)`. +So far we have left out an important aspect of a standard linear regression +model. Usually there is a constant bias term added to the model. Without +that, our model always crosses through the origin and we only learn the +right angle. An easy way to add such a bias term to our model is to add a +column of ones to the feature matrix `\(\mathbf{X}\)`. The corresponding weight in the parameter vector will then be the bias term. Here is how we add a bias column: diff --git a/website/Map_Reduce_Folder/tutorials/samsara/playing-with-samsara-flink-batch.md b/website/Map_Reduce_Folder/tutorials/samsara/playing-with-samsara-flink-batch.md index 158660b163..350ae7ac3a 100644 --- a/website/Map_Reduce_Folder/tutorials/samsara/playing-with-samsara-flink-batch.md +++ b/website/Map_Reduce_Folder/tutorials/samsara/playing-with-samsara-flink-batch.md @@ -1,13 +1,13 @@ --- layout: deprecated-docs -title: +title: redirect_from: - /documentation/tutorials/samsara/playing-with-samsara-flink-batch - /documentation/tutorials/samsara/playing-with-samsara-flink-batch.html --- -## Getting Started +## Getting Started To get started, add the following dependency to the pom: @@ -46,18 +46,18 @@ The top JIRA for Flink backend is [MAHOUT-1570](https://issues.apache.org/jira/b ### Implemented * [MAHOUT-1701](https://issues.apache.org/jira/browse/MAHOUT-1701) Mahout DSL for Flink: implement AtB ABt and AtA operators -* [MAHOUT-1702](https://issues.apache.org/jira/browse/MAHOUT-1702) implement element-wise operators (like `A + 2` or `A + B`) +* [MAHOUT-1702](https://issues.apache.org/jira/browse/MAHOUT-1702) implement element-wise operators (like `A + 2` or `A + B`) * [MAHOUT-1703](https://issues.apache.org/jira/browse/MAHOUT-1703) implement `cbind` and `rbind` * [MAHOUT-1709](https://issues.apache.org/jira/browse/MAHOUT-1709) implement slicing (like `A(1 to 10, ::)`) -* [MAHOUT-1710](https://issues.apache.org/jira/browse/MAHOUT-1710) implement right in-core matrix multiplication (`A %*% B` when `B` is in-core) +* [MAHOUT-1710](https://issues.apache.org/jira/browse/MAHOUT-1710) implement right in-core matrix multiplication (`A %*% B` when `B` is in-core) * [MAHOUT-1711](https://issues.apache.org/jira/browse/MAHOUT-1711) implement broadcasting * [MAHOUT-1712](https://issues.apache.org/jira/browse/MAHOUT-1712) implement operators `At`, `Ax`, `Atx` - `Ax` and `At` are implemented * [MAHOUT-1734](https://issues.apache.org/jira/browse/MAHOUT-1734) implement I/O - should be able to read results of Flink bindings * [MAHOUT-1747](https://issues.apache.org/jira/browse/MAHOUT-1747) add support for different types of indexes (String, long, etc) - now supports `Int`, `Long` and `String` -* [MAHOUT-1748](https://issues.apache.org/jira/browse/MAHOUT-1748) switch to Flink Scala API +* [MAHOUT-1748](https://issues.apache.org/jira/browse/MAHOUT-1748) switch to Flink Scala API * [MAHOUT-1749](https://issues.apache.org/jira/browse/MAHOUT-1749) Implement `Atx` * [MAHOUT-1750](https://issues.apache.org/jira/browse/MAHOUT-1750) Implement `ABt` -* [MAHOUT-1751](https://issues.apache.org/jira/browse/MAHOUT-1751) Implement `AtA` +* [MAHOUT-1751](https://issues.apache.org/jira/browse/MAHOUT-1751) Implement `AtA` * [MAHOUT-1755](https://issues.apache.org/jira/browse/MAHOUT-1755) Flush intermediate results to FS - Flink, unlike Spark, does not store intermediate results in memory. * [MAHOUT-1764](https://issues.apache.org/jira/browse/MAHOUT-1764) Add standard backend tests for Flink * [MAHOUT-1765](https://issues.apache.org/jira/browse/MAHOUT-1765) Add documentation about Flink backend @@ -81,14 +81,14 @@ The top JIRA for Flink backend is [MAHOUT-1570](https://issues.apache.org/jira/b * [MAHOUT-1824](https://issues.apache.org/jira/browse/MAHOUT-1824) Optimize FlinkOpAtA to use upper triangular matrices * [MAHOUT-1825](https://issues.apache.org/jira/browse/MAHOUT-1825) Add List of Flink algorithms to Mahout wiki page -### Tests +### Tests -There is a set of standard tests that all engines should pass (see [MAHOUT-1764](https://issues.apache.org/jira/browse/MAHOUT-1764)). +There is a set of standard tests that all engines should pass (see [MAHOUT-1764](https://issues.apache.org/jira/browse/MAHOUT-1764)). -* `DistributedDecompositionsSuite` -* `DrmLikeOpsSuite` -* `DrmLikeSuite` -* `RLikeDrmOpsSuite` +* `DistributedDecompositionsSuite` +* `DrmLikeOpsSuite` +* `DrmLikeSuite` +* `RLikeDrmOpsSuite` These are Flink-backend specific tests, e.g. @@ -98,14 +98,14 @@ These are Flink-backend specific tests, e.g. * `LATestSuite` tests for specific operators like `AtB`, `Ax`, etc * `UseCasesSuite` has more complex examples, like power iteration, ridge regression, etc -## Environment +## Environment -For development the minimal supported configuration is +For development the minimal supported configuration is * [JDK 1.7](http://www.oracle.com/technetwork/java/javase/downloads/jdk7-downloads-1880260.html) * [Scala 2.10] -When using mahout, please import the following modules: +When using mahout, please import the following modules: * `mahout-math` * `mahout-math-scala` diff --git a/website/Map_Reduce_Folder/tutorials/samsara/spark-naive-bayes.md b/website/Map_Reduce_Folder/tutorials/samsara/spark-naive-bayes.md index 452e1adc96..aa0959d587 100644 --- a/website/Map_Reduce_Folder/tutorials/samsara/spark-naive-bayes.md +++ b/website/Map_Reduce_Folder/tutorials/samsara/spark-naive-bayes.md @@ -14,23 +14,23 @@ redirect_from: Mahout currently has two flavors of Naive Bayes. The first is standard Multinomial Naive Bayes. The second is an implementation of Transformed Weight-normalized Complement Naive Bayes as introduced by Rennie et al. [[1]](http://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf). We refer to the former as Bayes and the latter as CBayes. -Where Bayes has long been a standard in text classification, CBayes is an extension of Bayes that performs particularly well on datasets with skewed classes and has been shown to be competitive with algorithms of higher complexity such as Support Vector Machines. +Where Bayes has long been a standard in text classification, CBayes is an extension of Bayes that performs particularly well on datasets with skewed classes and has been shown to be competitive with algorithms of higher complexity such as Support Vector Machines. ## Implementations -The mahout `math-scala` library has an implemetation of both Bayes and CBayes which is further optimized in the `spark` module. Currently the Spark optimized version provides CLI drivers for training and testing. Mahout Spark-Naive-Bayes models can also be trained, tested and saved to the filesystem from the Mahout Spark Shell. +The mahout `math-scala` library has an implemetation of both Bayes and CBayes which is further optimized in the `spark` module. Currently the Spark optimized version provides CLI drivers for training and testing. Mahout Spark-Naive-Bayes models can also be trained, tested and saved to the filesystem from the Mahout Spark Shell. ## Preprocessing and Algorithm -As described in [[1]](http://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf) Mahout Naive Bayes is broken down into the following steps (assignments are over all possible index values): +As described in [[1]](http://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf) Mahout Naive Bayes is broken down into the following steps (assignments are over all possible index values): - Let `\(\vec{d}=(\vec{d_1},...,\vec{d_n})\)` be a set of documents; `\(d_{ij}\)` is the count of word `\(i\)` in document `\(j\)`. - Let `\(\vec{y}=(y_1,...,y_n)\)` be their labels. -- Let `\(\alpha_i\)` be a smoothing parameter for all words in the vocabulary; let `\(\alpha=\sum_i{\alpha_i}\)`. +- Let `\(\alpha_i\)` be a smoothing parameter for all words in the vocabulary; let `\(\alpha=\sum_i{\alpha_i}\)`. - **Preprocessing**(via seq2Sparse) TF-IDF transformation and L2 length normalization of `\(\vec{d}\)` - 1. `\(d_{ij} = \sqrt{d_{ij}}\)` - 2. `\(d_{ij} = d_{ij}\left(\log{\frac{\sum_k1}{\sum_k\delta_{ik}+1}}+1\right)\)` - 3. `\(d_{ij} =\frac{d_{ij}}{\sqrt{\sum_k{d_{kj}^2}}}\)` + 1. `\(d_{ij} = \sqrt{d_{ij}}\)` + 2. `\(d_{ij} = d_{ij}\left(\log{\frac{\sum_k1}{\sum_k\delta_{ik}+1}}+1\right)\)` + 3. `\(d_{ij} =\frac{d_{ij}}{\sqrt{\sum_k{d_{kj}^2}}}\)` - **Training: Bayes**`\((\vec{d},\vec{y})\)` calculate term weights `\(w_{ci}\)` as: 1. `\(\hat\theta_{ci}=\frac{d_{ic}+\alpha_i}{\sum_k{d_{kc}}+\alpha}\)` 2. `\(w_{ci}=\log{\hat\theta_{ci}}\)` @@ -42,19 +42,19 @@ As described in [[1]](http://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf) 1. Let `\(\vec{t}= (t_1,...,t_n)\)` be a test document; let `\(t_i\)` be the count of the word `\(t\)`. 2. Label the document according to `\(l(t)=\arg\max_c \sum\limits_{i} t_i w_{ci}\)` -As we can see, the main difference between Bayes and CBayes is the weight calculation step. Where Bayes weighs terms more heavily based on the likelihood that they belong to class `\(c\)`, CBayes seeks to maximize term weights on the likelihood that they do not belong to any other class. +As we can see, the main difference between Bayes and CBayes is the weight calculation step. Where Bayes weighs terms more heavily based on the likelihood that they belong to class `\(c\)`, CBayes seeks to maximize term weights on the likelihood that they do not belong to any other class. ## Running from the command line -Mahout provides CLI drivers for all above steps. Here we will give a simple overview of Mahout CLI commands used to preprocess the data, train the model and assign labels to the training set. An [example script](https://github.com/apache/mahout/blob/master/examples/bin/classify-20newsgroups.sh) is given for the full process from data acquisition through classification of the classic [20 Newsgroups corpus](https://mahout.apache.org/users/classification/twenty-newsgroups.html). +Mahout provides CLI drivers for all above steps. Here we will give a simple overview of Mahout CLI commands used to preprocess the data, train the model and assign labels to the training set. An [example script](https://github.com/apache/mahout/blob/master/examples/bin/classify-20newsgroups.sh) is given for the full process from data acquisition through classification of the classic [20 Newsgroups corpus](https://mahout.apache.org/users/classification/twenty-newsgroups.html). - **Preprocessing:** For a set of Sequence File Formatted documents in PATH_TO_SEQUENCE_FILES the [mahout seq2sparse](https://mahout.apache.org/users/basics/creating-vectors-from-text.html) command performs the TF-IDF transformations (-wt tfidf option) and L2 length normalization (-n 2 option) as follows: - $ mahout seq2sparse - -i ${PATH_TO_SEQUENCE_FILES} - -o ${PATH_TO_TFIDF_VECTORS} - -nv + $ mahout seq2sparse + -i ${PATH_TO_SEQUENCE_FILES} + -o ${PATH_TO_TFIDF_VECTORS} + -nv -n 2 -wt tfidf @@ -62,73 +62,70 @@ For a set of Sequence File Formatted documents in PATH_TO_SEQUENCE_FILES the [ma The model is then trained using `mahout spark-trainnb`. The default is to train a Bayes model. The -c option is given to train a CBayes model: $ mahout spark-trainnb - -i ${PATH_TO_TFIDF_VECTORS} + -i ${PATH_TO_TFIDF_VECTORS} -o ${PATH_TO_MODEL} - -ow + -ow -c - **Label Assignment/Testing:** Classification and testing on a holdout set can then be performed via `mahout spark-testnb`. Again, the -c option indicates that the model is CBayes: - $ mahout spark-testnb + $ mahout spark-testnb -i ${PATH_TO_TFIDF_TEST_VECTORS} - -m ${PATH_TO_MODEL} - -c + -m ${PATH_TO_MODEL} + -c ## Command line options -- **Preprocessing:** *note: still reliant on MapReduce seq2sparse* - +- **Preprocessing:** *note: still reliant on MapReduce seq2sparse* + Only relevant parameters used for Bayes/CBayes as detailed above are shown. Several other transformations can be performed by `mahout seq2sparse` and used as input to Bayes/CBayes. For a full list of `mahout seq2Sparse` options see the [Creating vectors from text](https://mahout.apache.org/users/basics/creating-vectors-from-text.html) page. - $ mahout seq2sparse - --output (-o) output The directory pathname for output. - --input (-i) input Path to job input directory. - --weight (-wt) weight The kind of weight to use. Currently TF - or TFIDF. Default: TFIDF - --norm (-n) norm The norm to use, expressed as either a - float or "INF" if you want to use the - Infinite norm. Must be greater or equal - to 0. The default is not to normalize - --overwrite (-ow) If set, overwrite the output directory - --sequentialAccessVector (-seq) (Optional) Whether output vectors should - be SequentialAccessVectors. If set true - else false - --namedVector (-nv) (Optional) Whether output vectors should - be NamedVectors. If set true else false + $ mahout seq2sparse + --output (-o) output The directory pathname for output. + --input (-i) input Path to job input directory. + --weight (-wt) weight The kind of weight to use. Currently TF + or TFIDF. Default: TFIDF + --norm (-n) norm The norm to use, expressed as either a + float or "INF" if you want to use the + Infinite norm. Must be greater or equal + to 0. The default is not to normalize + --overwrite (-ow) If set, overwrite the output directory + --sequentialAccessVector (-seq) (Optional) Whether output vectors should + be SequentialAccessVectors. If set true + else false + --namedVector (-nv) (Optional) Whether output vectors should + be NamedVectors. If set true else false - **Training:** $ mahout spark-trainnb - --input (-i) input Path to job input directory. - --output (-o) output The directory pathname for output. + --input (-i) input Path to job input directory. + --output (-o) output The directory pathname for output. --trainComplementary (-c) Train complementary? Default is false. --master (-ma) Spark Master URL (optional). Default: "local". - Note that you can specify the number of - cores to get a performance improvement, + Note that you can specify the number of + cores to get a performance improvement, for example "local[4]" - --help (-h) Print out help + --help (-h) Print out help - **Testing:** - $ mahout spark-testnb - --input (-i) input Path to job input directory. - --model (-m) model The path to the model built during training. - --testComplementary (-c) Test complementary? Default is false. - --master (-ma) Spark Master URL (optional). Default: "local". - Note that you can specify the number of - cores to get a performance improvement, - for example "local[4]" - --help (-h) Print out help + $ mahout spark-testnb + --input (-i) input Path to job input directory. + --model (-m) model The path to the model built during training. + --testComplementary (-c) Test complementary? Default is false. + --master (-ma) Spark Master URL (optional). Default: "local". + Note that you can specify the number of + cores to get a performance improvement, + for example "local[4]" + --help (-h) Print out help ## Examples 1. [20 Newsgroups classification](https://github.com/apache/mahout/blob/master/examples/bin/classify-20newsgroups.sh) 2. [Document classification with Naive Bayes in the Mahout shell](https://github.com/apache/mahout/blob/master/examples/bin/spark-document-classifier.mscala) - - -## References - -[1]: Jason D. M. Rennie, Lawerence Shih, Jamie Teevan, David Karger (2003). [Tackling the Poor Assumptions of Naive Bayes Text Classifiers](http://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf). Proceedings of the Twentieth International Conference on Machine Learning (ICML-2003). +## References +[1]: Jason D. M. Rennie, Lawerence Shih, Jamie Teevan, David Karger (2003). [Tackling the Poor Assumptions of Naive Bayes Text Classifiers](http://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf). Proceedings of the Twentieth International Conference on Machine Learning (ICML-2003). diff --git a/website/_config.yml b/website/_config.yml index ad539a9aff..0bdddcfff2 100644 --- a/website/_config.yml +++ b/website/_config.yml @@ -20,7 +20,7 @@ title : Apache Mahout description: Distributed Linear Algebra -mahout-version: 0.4 +mahout-version: 0.4 author : name : The Apache Software Foundation @@ -62,4 +62,3 @@ exclude: collections: papers: output: true - diff --git a/website/_includes/sidebar.html b/website/_includes/sidebar.html index 2e2c6fbcdf..368e551aa8 100644 --- a/website/_includes/sidebar.html +++ b/website/_includes/sidebar.html @@ -31,7 +31,7 @@
  • FOSSY 2024, Portland Oregon - QuMat: Apache Mahout's Quantum Computing Interface - + [Slides]
  • diff --git a/website/_papers/An-Efficient-Quantum-Factoring-Algorithm.md b/website/_papers/An-Efficient-Quantum-Factoring-Algorithm.md index ffbd19706d..9a229f29e1 100644 --- a/website/_papers/An-Efficient-Quantum-Factoring-Algorithm.md +++ b/website/_papers/An-Efficient-Quantum-Factoring-Algorithm.md @@ -8,30 +8,30 @@ Author: Oded Regev [Original Paper](https://arxiv.org/abs/2308.06572) -The paper presents an efficient quantum factoring algorithm that can be used to -factorize n-bit integers. The algorithm involves running a quantum circuit with -˜O(n3/2) gates for √n + 4 times, and then using a polynomial-time classical -post-processing step. The correctness of the algorithm is based on a -number-theoretic assumption similar to those used in subexponential classical -factorization algorithms. The author demonstrates that quantum circuits of size -˜O(n3/2) are sufficient for factoring integers, which is an improvement over -previous algorithms that required larger circuit sizes. The number of qubits in -the quantum circuit is O(n3/2), which is higher than the qubit requirement in +The paper presents an efficient quantum factoring algorithm that can be used to +factorize n-bit integers. The algorithm involves running a quantum circuit with +˜O(n3/2) gates for √n + 4 times, and then using a polynomial-time classical +post-processing step. The correctness of the algorithm is based on a +number-theoretic assumption similar to those used in subexponential classical +factorization algorithms. The author demonstrates that quantum circuits of size +˜O(n3/2) are sufficient for factoring integers, which is an improvement over +previous algorithms that required larger circuit sizes. The number of qubits in +the quantum circuit is O(n3/2), which is higher than the qubit requirement in optimized implementations of Shor's algorithm. However, the depth of the quantum -circuit is smaller than Shor's algorithm, making it more feasible for -implementation. The paper also discusses the potential implications of the +circuit is smaller than Shor's algorithm, making it more feasible for +implementation. The paper also discusses the potential implications of the algorithm in practice. It is highlighted that the analysis is asymptotic and the -algorithm may not be efficient for small values of n. The algorithm may benefit -from optimizations in fast integer multiplication and the use of smaller qubit -counts, similar to optimizations used in Shor's algorithm. However, it is -currently unclear if these optimizations can be applied to the proposed -algorithm. The author concludes by stating that the algorithm provides an -improvement over Shor's algorithm in terms of circuit size. However, it remains +algorithm may not be efficient for small values of n. The algorithm may benefit +from optimizations in fast integer multiplication and the use of smaller qubit +counts, similar to optimizations used in Shor's algorithm. However, it is +currently unclear if these optimizations can be applied to the proposed +algorithm. The author concludes by stating that the algorithm provides an +improvement over Shor's algorithm in terms of circuit size. However, it remains to be seen if the algorithm can be practically implemented and if it can provide -an improvement over Shor's algorithm for small values of n. The analysis in the -paper is based on asymptotics, and it is unclear if hidden constants in the +an improvement over Shor's algorithm for small values of n. The analysis in the +paper is based on asymptotics, and it is unclear if hidden constants in the algorithm would make it inefficient for small values of n. In summary, the paper -presents an efficient quantum factoring algorithm that uses a quantum circuit +presents an efficient quantum factoring algorithm that uses a quantum circuit with ˜O(n3/2) gates and a classical post-processing step. The algorithm provides -an improvement over previous algorithms in terms of circuit size, but its +an improvement over previous algorithms in terms of circuit size, but its practicality and potential improvements for small values of n remain to be seen. diff --git a/website/_papers/Quantum-Kernel-Estimation-With-Neutral-Atoms-For-Supervised-Classification.md b/website/_papers/Quantum-Kernel-Estimation-With-Neutral-Atoms-For-Supervised-Classification.md index e6c4598219..3af8bf7a93 100644 --- a/website/_papers/Quantum-Kernel-Estimation-With-Neutral-Atoms-For-Supervised-Classification.md +++ b/website/_papers/Quantum-Kernel-Estimation-With-Neutral-Atoms-For-Supervised-Classification.md @@ -8,36 +8,36 @@ Author: Marco Russo, Edoardo Giusto, Bartolomeo Montrucchio [Original Paper](https://arxiv.org/abs/2307.15840) -In this paper, the authors propose a gate-based approach to quantum kernel -estimation (QKE) for supervised classification using neutral atom quantum -computers. QKE is a technique that leverages the power of quantum computing to -estimate a kernel function that is difficult to compute classically. The -estimated kernel is then used by a classical computer to train a support vector -machine (SVM) for classification tasks. The authors focus on neutral atom -quantum computers because they allow for more freedom in arranging the atoms, -which is essential for implementing the necessary gates for QKE. They present a -general method for deriving 1-qubit and 2-qubit gates from laser pulses, which -are then used to construct a parameterized sequence for feature mapping on 3 -qubits. They show that this approach can be extended to N qubits, taking +In this paper, the authors propose a gate-based approach to quantum kernel +estimation (QKE) for supervised classification using neutral atom quantum +computers. QKE is a technique that leverages the power of quantum computing to +estimate a kernel function that is difficult to compute classically. The +estimated kernel is then used by a classical computer to train a support vector +machine (SVM) for classification tasks. The authors focus on neutral atom +quantum computers because they allow for more freedom in arranging the atoms, +which is essential for implementing the necessary gates for QKE. They present a +general method for deriving 1-qubit and 2-qubit gates from laser pulses, which +are then used to construct a parameterized sequence for feature mapping on 3 +qubits. They show that this approach can be extended to N qubits, taking advantage of the more flexible arrangement of atoms in neutral atom devices. The -experimental setup involves simulating the Pasqal Chadoq2 device, which allows -for planar arrangement of atoms. The authors generate a dataset of 40 training -samples and 20 test samples with 3 features and a separation gap of 0.1. They +experimental setup involves simulating the Pasqal Chadoq2 device, which allows +for planar arrangement of atoms. The authors generate a dataset of 40 training +samples and 20 test samples with 3 features and a separation gap of 0.1. They use the Qiskit library to implement the feature mapping circuit and generate the -sequences of pulses for QKE. The training and testing of the SVM are performed -on a classical computer using the estimated kernel matrices. The results show -that the proposed approach achieves a high accuracy of 75% on the test set, -despite the small size of the dataset and the low separation. The authors -compare the performance to a classical SVM with a radial basis function kernel -and find that the quantum approach outperforms the classical approach. The -authors discuss the advantages of using neutral atom quantum computers for QKE. -The arbitrary arrangement of atoms allows for more direct connections between -qubits, reducing the depth of the circuit and reducing the impact of -decoherence. They also highlight the exponential computational advantage of -quantum feature kernels over classical kernel computation methods for -high-dimensional feature spaces. Overall, the paper presents a gate-based -approach to QKE using neutral atom quantum computers. The experimental results -demonstrate the potential of this approach for supervised classification tasks -and highlight the advantages of neutral atom devices for implementing QKE -circuits. The paper provides a foundation for future research in the field of +sequences of pulses for QKE. The training and testing of the SVM are performed +on a classical computer using the estimated kernel matrices. The results show +that the proposed approach achieves a high accuracy of 75% on the test set, +despite the small size of the dataset and the low separation. The authors +compare the performance to a classical SVM with a radial basis function kernel +and find that the quantum approach outperforms the classical approach. The +authors discuss the advantages of using neutral atom quantum computers for QKE. +The arbitrary arrangement of atoms allows for more direct connections between +qubits, reducing the depth of the circuit and reducing the impact of +decoherence. They also highlight the exponential computational advantage of +quantum feature kernels over classical kernel computation methods for +high-dimensional feature spaces. Overall, the paper presents a gate-based +approach to QKE using neutral atom quantum computers. The experimental results +demonstrate the potential of this approach for supervised classification tasks +and highlight the advantages of neutral atom devices for implementing QKE +circuits. The paper provides a foundation for future research in the field of quantum machine learning and quantum computing. diff --git a/website/_papers/Quantum-machine-learning-beyond-kernel-methods.md b/website/_papers/Quantum-machine-learning-beyond-kernel-methods.md index 525c516604..be3d939923 100644 --- a/website/_papers/Quantum-machine-learning-beyond-kernel-methods.md +++ b/website/_papers/Quantum-machine-learning-beyond-kernel-methods.md @@ -10,19 +10,19 @@ Journal: Nature Communications [Original Paper](https://www.nature.com/articles/s41467-023-36159-y) -This article presents a study on quantum machine learning models and their -comparison to classical models. The authors propose a framework that captures -all standard models based on parametrized quantum circuits, focusing on linear -quantum models. They analyze the resource requirements and learning performance +This article presents a study on quantum machine learning models and their +comparison to classical models. The authors propose a framework that captures +all standard models based on parametrized quantum circuits, focusing on linear +quantum models. They analyze the resource requirements and learning performance guarantees of these models, particularly comparing explicit and implicit models. -They show that implicit models can achieve a lower training loss but may suffer -from poor generalization performance. They also show that data re-uploading -models, a type of explicit model, can be more general than both explicit and -implicit models. The authors further investigate the advantages of explicit -models by testing their performance on a learning task involving -quantum-generated data. They find that explicit models can outperform both -implicit models and classical models on this task, highlighting the potential -learning advantage of explicit quantum models. The study provides insights into -the capabilities and limitations of different quantum machine learning models, +They show that implicit models can achieve a lower training loss but may suffer +from poor generalization performance. They also show that data re-uploading +models, a type of explicit model, can be more general than both explicit and +implicit models. The authors further investigate the advantages of explicit +models by testing their performance on a learning task involving +quantum-generated data. They find that explicit models can outperform both +implicit models and classical models on this task, highlighting the potential +learning advantage of explicit quantum models. The study provides insights into +the capabilities and limitations of different quantum machine learning models, and it contributes to understanding the possible advantages of quantum models in practical applications. diff --git a/website/_papers/Unleashing-the-Potential-of-LLMs-for-Quantum-Computing.md b/website/_papers/Unleashing-the-Potential-of-LLMs-for-Quantum-Computing.md index 6ed1a26d5d..5812c87e1e 100644 --- a/website/_papers/Unleashing-the-Potential-of-LLMs-for-Quantum-Computing.md +++ b/website/_papers/Unleashing-the-Potential-of-LLMs-for-Quantum-Computing.md @@ -4,46 +4,46 @@ title: "Summary of 'Unleashing the Potential of LLMs for Quantum Computing: A St date: 2024-03-04 --- -Author: Zhiding Liang, Jinglei Cheng, Rui Yang, Hang Ren, Zhixin Song, Di Wu, +Author: Zhiding Liang, Jinglei Cheng, Rui Yang, Hang Ren, Zhixin Song, Di Wu, Xuehai Qian, Tongyang Li, Yiyu Shi [Original Paper](https://arxiv.org/abs/2307.08191) This paper discusses the potential of large language models (LLMs), specifically -generative pretrained transformers (GPTs), in the field of quantum computing. -The authors propose a Quantum GPT-Guided Architecture Search (QGAS) model that -utilizes GPT-4 to recommend high-quality ansatz architectures for variational -quantum algorithms (VQAs). The ansatz architecture is a crucial component of -quantum computing and determines the efficiency and accuracy of quantum -algorithms. The authors conduct experiments using a series of application -benchmarks, including portfolio optimization, the MaxCut problem, the Traveling -Salesman Problem (TSP), and the estimation of molecule ground state energy for -Lithium Hydride (LiH) and Water (H2O). They compare the performance of the -ansatz architectures generated by QGAS with existing ansatzes and -state-of-the-art ansatz architecture search methods. The results show that QGAS -outperforms other ansatz architectures in some benchmark applications, -demonstrating the potential of LLMs in quantum architecture design. The authors -highlight the importance of human feedback in guiding the performance of GPT-4. -Human experts provide specific guidance and feedback to improve the search -strategies and evaluate the generated ansatz architectures. The iterative -feedback loop between human experts and GPT-4 leads to better performance and -optimization of the quantum circuits. The paper also discusses the limitations -of GPT in the field of quantum computing. GPT is not a general artificial +generative pretrained transformers (GPTs), in the field of quantum computing. +The authors propose a Quantum GPT-Guided Architecture Search (QGAS) model that +utilizes GPT-4 to recommend high-quality ansatz architectures for variational +quantum algorithms (VQAs). The ansatz architecture is a crucial component of +quantum computing and determines the efficiency and accuracy of quantum +algorithms. The authors conduct experiments using a series of application +benchmarks, including portfolio optimization, the MaxCut problem, the Traveling +Salesman Problem (TSP), and the estimation of molecule ground state energy for +Lithium Hydride (LiH) and Water (H2O). They compare the performance of the +ansatz architectures generated by QGAS with existing ansatzes and +state-of-the-art ansatz architecture search methods. The results show that QGAS +outperforms other ansatz architectures in some benchmark applications, +demonstrating the potential of LLMs in quantum architecture design. The authors +highlight the importance of human feedback in guiding the performance of GPT-4. +Human experts provide specific guidance and feedback to improve the search +strategies and evaluate the generated ansatz architectures. The iterative +feedback loop between human experts and GPT-4 leads to better performance and +optimization of the quantum circuits. The paper also discusses the limitations +of GPT in the field of quantum computing. GPT is not a general artificial intelligence and cannot think dynamically about quantum physics or make accurate predictions about scientific phenomena in quantum experiments. It also relies on -large-scale data models, which may contain biased or misleading information -about quantum computing. The authors suggest future directions for the -integration of LLMs, such as GPT, in quantum computing. They propose that GPT -can be used to design and optimize fault-tolerant quantum algorithms and assist +large-scale data models, which may contain biased or misleading information +about quantum computing. The authors suggest future directions for the +integration of LLMs, such as GPT, in quantum computing. They propose that GPT +can be used to design and optimize fault-tolerant quantum algorithms and assist in the calibration of quantum hardware. They also envision GPT playing a role in -the simulation of quantum computers and providing agile validation of -algorithmic innovations. In conclusion, this paper highlights the potential of -LLMs, specifically GPT, in the field of quantum computing. The QGAS model -demonstrates the effectiveness of using GPT-4 to generate high-performance -ansatz architectures for quantum algorithms. The integration of human feedback -and the power of GPT-4 provides a promising avenue for advancing quantum -architecture design and optimization. However, the limitations of GPT and the -challenges of applying LLMs to quantum computing should be considered. The +the simulation of quantum computers and providing agile validation of +algorithmic innovations. In conclusion, this paper highlights the potential of +LLMs, specifically GPT, in the field of quantum computing. The QGAS model +demonstrates the effectiveness of using GPT-4 to generate high-performance +ansatz architectures for quantum algorithms. The integration of human feedback +and the power of GPT-4 provides a promising avenue for advancing quantum +architecture design and optimization. However, the limitations of GPT and the +challenges of applying LLMs to quantum computing should be considered. The authors suggest further research and development to leverage the capabilities of -GPT and address the limitations to fully harness the potential of LLMs in +GPT and address the limitations to fully harness the potential of LLMs in quantum computing. diff --git a/website/_posts/2017-12-20-new-website.md b/website/_posts/2017-12-20-new-website.md index 81f9777cb5..e1cf317486 100644 --- a/website/_posts/2017-12-20-new-website.md +++ b/website/_posts/2017-12-20-new-website.md @@ -7,7 +7,7 @@ category: news We've launced a new website, (a HUUUGE special thanks to David Miller from [StartBootstrap.com](http://startbootstrap.com)). -It looks great but there's lots to do as we migrate and clean up content from the old site. If you see anything broken, +It looks great but there's lots to do as we migrate and clean up content from the old site. If you see anything broken, please let us know via either: - send an email to dev@mahout.apache.org -- fix it yourself and [open a PR](/documentation/developers/githubPRs)! +- fix it yourself and [open a PR](/documentation/developers/githubPRs)! diff --git a/website/_posts/2018-06-18-preping-014.md b/website/_posts/2018-06-18-preping-014.md index 90b902f0ff..fa25850437 100644 --- a/website/_posts/2018-06-18-preping-014.md +++ b/website/_posts/2018-06-18-preping-014.md @@ -7,22 +7,21 @@ category: news Hey all! We're preparing for Mahout version 14.1. -This release might not seem super exciting from a user perspective (except +This release might not seem super exciting from a user perspective (except we're bumping Apache Spark to version 2.x and Scala to version 2.11) but -it includes some very important 'behind the scenes changes'. We're doing a -heavy refactoring which will combine `math` and `math-scala` into a module +it includes some very important 'behind the scenes changes'. We're doing a +heavy refactoring which will combine `math` and `math-scala` into a module called `core` (no change to code required!). -We will be moving H2o and Flink-batch engines into `community` that is to -say the code will still be available, but we won't be actively maintaining +We will be moving H2o and Flink-batch engines into `community` that is to +say the code will still be available, but we won't be actively maintaining it (we will accept PRs though). We're moving the Map Reduce code to community and have considered to once again accepting PRs against this code (if you have an open PR, we'll hit you up after the release about rebasing and considering to resubmit the PR) -We're going to be updating the Java dependency from 1.7 to 1.8. +We're going to be updating the Java dependency from 1.7 to 1.8. There was a LOT of old junk in the POMs which we have clipped out. -Stay tuned and [feel free to help out](https://issues.apache.org/jira/secure/RapidBoard.jspa?rapidView=80&projectKey=MAHOUT&view=detail&selectedIssue=MAHOUT-2047)! - +Stay tuned and [feel free to help out](https://issues.apache.org/jira/secure/RapidBoard.jspa?rapidView=80&projectKey=MAHOUT&view=detail&selectedIssue=MAHOUT-2047)! diff --git a/website/_posts/2019-12-14-version-14-1.md b/website/_posts/2019-12-14-version-14-1.md index d048fea2cc..4f7c74885c 100644 --- a/website/_posts/2019-12-14-version-14-1.md +++ b/website/_posts/2019-12-14-version-14-1.md @@ -8,13 +8,13 @@ category: news ### On wierd version numbering... -We were under pressure to release a "version above 1.0" which seemed sort of arbitrary to us. +We were under pressure to release a "version above 1.0" which seemed sort of arbitrary to us. We decided to just stick with our original schema, but take it x10. Our last release was 0.14.0, and thus this release is 14.1. So if you're wondering where versions 1.0-13.0 are, that's what happened. 14.1 is a long overdue fix for us releasing binaries for to use, instead of having to build your own. 0.14.0 was us finally -moving the old Map Reduce stuff out of the main trunk, because we haven't been about that for quite some time. +moving the old Map Reduce stuff out of the main trunk, because we haven't been about that for quite some time. If you would like to help us test the release candidate, send an email to `dev-subscribe@mahout.apache.org` and reply to the confimration email to join our mailing list, and happy Mahouting! diff --git a/website/_posts/2020-10-18-version-14-1-released.md b/website/_posts/2020-10-18-version-14-1-released.md index 8091362357..2c8819b47a 100644 --- a/website/_posts/2020-10-18-version-14-1-released.md +++ b/website/_posts/2020-10-18-version-14-1-released.md @@ -5,9 +5,8 @@ date: 2020-10-18 16:16:01 -0600 category: news --- -Oh happy day! A lot of work went into this release with getting the build system to work again so that we can release +Oh happy day! A lot of work went into this release with getting the build system to work again so that we can release binaries. There really weren't too many new features over 14.0, but binaries now exist in Maven. -On another note, we had a really exciting "MahoutCon" track at ApacheCon@home this year. A special thanks to all of our -speakers, and we plan to post the videos here when we're able. - +On another note, we had a really exciting "MahoutCon" track at ApacheCon@home this year. A special thanks to all of our +speakers, and we plan to post the videos here when we're able. diff --git a/website/_posts/2020-10-30-weekly-meeting-minutes.md b/website/_posts/2020-10-30-weekly-meeting-minutes.md index 640d617593..cd19459c18 100644 --- a/website/_posts/2020-10-30-weekly-meeting-minutes.md +++ b/website/_posts/2020-10-30-weekly-meeting-minutes.md @@ -9,7 +9,7 @@ category: minutes Trevor has been doing some serious overhauling on the Apache Mahout entry on Wikipedia. -However, he'd appreciate it if people could fill out sections, especially if they have expertise. +However, he'd appreciate it if people could fill out sections, especially if they have expertise. Further- we would like to add architecture diagrams- both the original and the one with the native solver- need to ask what the copyright it on those/if we can use them. @@ -21,6 +21,6 @@ The Apache Zeppelin+Mahout docker image is live- you can get started with Mahout ### Topic: Starting on Python Bindings -Trevor is going to start working on Python bindings. He thinks adding Mahout as a library for pyspark -will significantly increase adoption. If anyone has experience using py4j or is otherwise interested in -helping or wants to give words of encouragement reach out on dev or email him directly. +Trevor is going to start working on Python bindings. He thinks adding Mahout as a library for pyspark +will significantly increase adoption. If anyone has experience using py4j or is otherwise interested in +helping or wants to give words of encouragement reach out on dev or email him directly. diff --git a/website/_posts/2020-11-13-weekly-meeting-minutes.md b/website/_posts/2020-11-13-weekly-meeting-minutes.md index 82e2f5cc7b..d5b4010082 100644 --- a/website/_posts/2020-11-13-weekly-meeting-minutes.md +++ b/website/_posts/2020-11-13-weekly-meeting-minutes.md @@ -9,7 +9,7 @@ Weekly community meeting minutes. ### Collected Videos / Conference Talks -Andrew Musselman is going to try to work on MAHOUT-2130 (Build talks page) this weekend. If you have / know of any +Andrew Musselman is going to try to work on MAHOUT-2130 (Build talks page) this weekend. If you have / know of any content, please send it to him or just add it after he launches the page. ### Issue Grooming @@ -17,13 +17,11 @@ content, please send it to him or just add it after he launches the page. The following issues are all related - MAHOUT-2122 Create getting started on Docker page -- MAHOUT-2134 Update Mahout in Zeppelin page +- MAHOUT-2134 Update Mahout in Zeppelin page - MAHOUT-2129 Rebuild website docs, add Getting started with Zeppelin to Nav - MAHOUT-2126 Update Zeppelin Containers -- MAHOUT-2133 Update "Getting Started in Mahout" page. - -The narrative is- we should update (or replace) the getting started in Mahout page with Getting Started with Mahout in -Zeppelin. The Zeppelin Docker should have some more "Getting Started" Notebooks. The current Mahout in Zeppelin page -should point to the new Mahout in Zeppelin on Docker page. The nav should be updated to reflect all of this. - +- MAHOUT-2133 Update "Getting Started in Mahout" page. +The narrative is- we should update (or replace) the getting started in Mahout page with Getting Started with Mahout in +Zeppelin. The Zeppelin Docker should have some more "Getting Started" Notebooks. The current Mahout in Zeppelin page +should point to the new Mahout in Zeppelin on Docker page. The nav should be updated to reflect all of this. diff --git a/website/_posts/2020-11-20-weekly-meeting-minutes.md b/website/_posts/2020-11-20-weekly-meeting-minutes.md index 231bbfea6f..74063ddee3 100644 --- a/website/_posts/2020-11-20-weekly-meeting-minutes.md +++ b/website/_posts/2020-11-20-weekly-meeting-minutes.md @@ -7,6 +7,6 @@ category: minutes Weekly community meeting minutes. -No new issues. +No new issues. Will propose idea of moving meeting time on dev@m.a.o diff --git a/website/_posts/2020-11-6-weekly-meeting-minutes.md b/website/_posts/2020-11-6-weekly-meeting-minutes.md index c8857440b2..9e34cc682d 100644 --- a/website/_posts/2020-11-6-weekly-meeting-minutes.md +++ b/website/_posts/2020-11-6-weekly-meeting-minutes.md @@ -25,6 +25,6 @@ No progress to date- weather is unseasonably nice in Chicago. -Created +Created - MAHOUT-2133 Update Quickstart Page - MAHOUT-2134 Update Zeppelin Tutorial diff --git a/website/_posts/2020-12-15-weekly-meeting-minutes.md b/website/_posts/2020-12-15-weekly-meeting-minutes.md index c77f73154d..61b1ab5fc6 100644 --- a/website/_posts/2020-12-15-weekly-meeting-minutes.md +++ b/website/_posts/2020-12-15-weekly-meeting-minutes.md @@ -7,4 +7,3 @@ category: minutes Trevor able to do Mahout in-core matrix addition in Python. Will continue working, but good first step. Want to see POC that DRMs will also "work" and then will sketch plan. - diff --git a/website/_posts/2020-12-8-weekly-meeting-minutes.md b/website/_posts/2020-12-8-weekly-meeting-minutes.md index 7550ca6613..6e880dc835 100644 --- a/website/_posts/2020-12-8-weekly-meeting-minutes.md +++ b/website/_posts/2020-12-8-weekly-meeting-minutes.md @@ -11,9 +11,9 @@ Musselman to continue work on 2130 (adding talks page). Trevor making (slow) progress wtih Py4J / Numpy / Python bindings (life getting in the way, but seems doable). -Palumbo interested in FPGA- +Palumbo interested in FPGA- > Investigation: FPGA4Mahout, Mahout BLAS Subroutines on FPGA. we are considering integration of the fBLAS library for FPGA acceleration of BLAS subroutines on mahout. fBLAS accepts JSON expressions of algebraic subroutines and generates OpenCL code with Just In Time compilation and FPGA flashing. We are first considering a Naive approach upon evaluation to convert the computation graph of a mahout expression into JSON, compile and Flash the FPGA and stream data through the FPGA in SIMD vectored fashion. -> +> > The fBLAS library shows significant speedups over CPU BLAS operations. If we find significant performance gains, In conjunction with the current and ongoing effort to implement Python Bindings for Mahout, this could attract HPC developers with needs for near-real-time computation, by providing Python and Java bindings for FPGA accelerated matrix algebra available for distributed and in-core math. > > An end goal of this effort would be to provide scientists, engineers, and others with little to no Hardware experience a Zeppelin Notebook on which they could can develop models or circuits, using the mahout DSL, while concurrntly programming the model onto an FPGA" diff --git a/website/_posts/2021-01-05-weekly-meeting-minutes.md b/website/_posts/2021-01-05-weekly-meeting-minutes.md index 15ec83e7e2..987a91a2d1 100644 --- a/website/_posts/2021-01-05-weekly-meeting-minutes.md +++ b/website/_posts/2021-01-05-weekly-meeting-minutes.md @@ -5,13 +5,12 @@ date: 2021-01-05 16:00:01 -0600 category: minutes --- -Andrew showed Trevor how to file his first board report. +Andrew showed Trevor how to file his first board report. Springer Publishing has shown interest in a second edition of the Encyclopedia of Big Data, and Musselman plans to update -the chapter on Mahout. +the chapter on Mahout. -Trevor needs to make a git repo to share with his experiments of Python+Mahout. +Trevor needs to make a git repo to share with his experiments of Python+Mahout. Reminder- community calls are open to all. We'll try to get a good link and post it to one of the minutes. In the mean time join dev@mahout.apache.org for the weekly link. - diff --git a/website/_posts/2021-02-02-weekly-meeting-minutes.md b/website/_posts/2021-02-02-weekly-meeting-minutes.md index 5cf0127f0d..9b279aa703 100644 --- a/website/_posts/2021-02-02-weekly-meeting-minutes.md +++ b/website/_posts/2021-02-02-weekly-meeting-minutes.md @@ -8,11 +8,10 @@ category: minutes ### Recap Since Last Time * Website build working again -* josehernandezsc added Ridge Regression to `algorithms/` framework. +* josehernandezsc added Ridge Regression to `algorithms/` framework. * Trevor shared incore PyMahout git repo on mailing list- needs to POC for distributed. ### Projects going forward * Still need to do 5 related issues identified [November 13th](http://mahout.apache.org/minutes/2020/11/13/weekly-meeting-minutes.html) * Trevor to POC distribute Mahout in Python - diff --git a/website/_posts/2021-06-01-Zeppelin Quickstart.md b/website/_posts/2021-06-01-Zeppelin Quickstart.md index 7992bba71a..60af5f7b52 100644 --- a/website/_posts/2021-06-01-Zeppelin Quickstart.md +++ b/website/_posts/2021-06-01-Zeppelin Quickstart.md @@ -8,12 +8,12 @@ category: tutorial # Apache Zeppelin Quickstart Apache Mahout is for Mathematicians (or statisticians, or data scientists, etc.). We understand not all users -are up to cloning and building from source, working with command line interfaces, and other such engineering -tasks. +are up to cloning and building from source, working with command line interfaces, and other such engineering +tasks. In that thread, we've created [The Apache Zeppelin Quickstart Docker Container](http://mahout.apache.org/docs/latest/tutorials/misc/getting-started-with-zeppelin/). -We hope this will make it easier for users to dip their toes in the Mahout waters. Let us know what you think on +We hope this will make it easier for users to dip their toes in the Mahout waters. Let us know what you think on * user at mahout dot apache dot org * dev at mahout dot apache dot org diff --git a/website/_posts/2023-02-22-Meeting-Minutes.md b/website/_posts/2023-02-22-Meeting-Minutes.md index 590529525d..dd9896e1f6 100644 --- a/website/_posts/2023-02-22-Meeting-Minutes.md +++ b/website/_posts/2023-02-22-Meeting-Minutes.md @@ -32,7 +32,7 @@ Notes from previous session * Apr: 14.2, point release with minor improvements and new build system * [Already done](https://mahout.apache.org/docs/latest/tutorials/misc/getting-started-with-zeppelin/) ~~Apr-Jun: Docker images (Zeppelin + Mahout, e.g.)~~ -### Ideas +### Ideas * Python bindings for DSL (XXXL) * Automating or maintaining Docker images (XL, including docs) * New parsing and processing modules @@ -40,7 +40,7 @@ Notes from previous session 1. Proof of concept GPL/BSD/ALv2/MIT classifier (start with small tutorial) * Live code environment (in-browser) to try DSL (XXL for a real version, start with simple env) -### Summary and next steps +### Summary and next steps * Continue monthly community calls * Complete JIRA focus * Get travis migration done, nightlies running diff --git a/website/_posts/2023-03-29-Meeting-Minutes.md b/website/_posts/2023-03-29-Meeting-Minutes.md index cce7645aae..e95d31f8a9 100644 --- a/website/_posts/2023-03-29-Meeting-Minutes.md +++ b/website/_posts/2023-03-29-Meeting-Minutes.md @@ -34,7 +34,7 @@ Round the room quick introductions ### Release plans * Apr: 14.2, point release with minor improvements and new build system -### Ideas +### Ideas * Python bindings for DSL (XXXL) * Automating or maintaining Docker images (XL, including docs) * New parsing and processing modules @@ -50,7 +50,7 @@ Round the room quick introductions ### Project management * Inactive PMC -### Summary and next steps +### Summary and next steps * Continue monthly community calls * Complete JIRA focus * Get travis migration done, nightlies running diff --git a/website/_posts/2023-07-26-Meeting-Minutes.md b/website/_posts/2023-07-26-Meeting-Minutes.md index 771d20e536..adb684e76f 100644 --- a/website/_posts/2023-07-26-Meeting-Minutes.md +++ b/website/_posts/2023-07-26-Meeting-Minutes.md @@ -30,7 +30,7 @@ We caught up on personal and ASF topics ### Release plans * Aug: 14.2, point release with minor improvements and new build system -### Ideas +### Ideas * Quantum computing (IBM and Honeywell have free-tier APIs) XXXL * Python bindings for DSL (XXXL) * Automating or maintaining Docker images (XL, including docs) @@ -47,7 +47,7 @@ We caught up on personal and ASF topics ### Project management * Previous vote on PMC involvement passed: https://lists.apache.org/thread/stv6jjjx72gmgs3pg7lvwf1wmngmtnbs -### Summary and next steps +### Summary and next steps * Continue monthly community calls * Complete JIRA focus * Get travis migration done, nightlies running diff --git a/website/_posts/2023-08-30-Meeting-Minutes.md b/website/_posts/2023-08-30-Meeting-Minutes.md index 77ed234a66..e8a9e259ea 100644 --- a/website/_posts/2023-08-30-Meeting-Minutes.md +++ b/website/_posts/2023-08-30-Meeting-Minutes.md @@ -30,7 +30,7 @@ category: minutes ### Release plans * Aug: 14.2, point release with minor improvements and new build system -### Ideas +### Ideas * Quantum computing (IBM and Honeywell have free-tier APIs) XXXL * Python bindings for DSL (XXXL) * Automating or maintaining Docker images (XL, including docs) @@ -47,7 +47,7 @@ category: minutes ### Project management * Previous vote on PMC involvement passed: https://lists.apache.org/thread/stv6jjjx72gmgs3pg7lvwf1wmngmtnbs -### Summary and next steps +### Summary and next steps * Continue monthly community calls * Complete JIRA focus * Get travis migration done, nightlies running diff --git a/website/_posts/2023-09-27-Meeting-Minutes.md b/website/_posts/2023-09-27-Meeting-Minutes.md index 97c01cb730..6b4417eed8 100644 --- a/website/_posts/2023-09-27-Meeting-Minutes.md +++ b/website/_posts/2023-09-27-Meeting-Minutes.md @@ -31,7 +31,7 @@ category: minutes ### Release plans * Nov: 14.2, point release with minor improvements and new build system -### Ideas +### Ideas * Quantum computing (IBM and Honeywell have free-tier APIs) XXXL * Python bindings for DSL (XXXL) * Automating or maintaining Docker images (XL, including docs) @@ -45,7 +45,7 @@ category: minutes * Look at OpenNLP for tokenizing and other processing * New embedding similarity job -### Summary and next steps +### Summary and next steps * Continue monthly community calls * Complete JIRA focus * Get travis migration done, nightlies running diff --git a/website/_posts/2023-10-25-Meeting-Minutes.md b/website/_posts/2023-10-25-Meeting-Minutes.md index b11a9467e3..da9742fb14 100644 --- a/website/_posts/2023-10-25-Meeting-Minutes.md +++ b/website/_posts/2023-10-25-Meeting-Minutes.md @@ -31,7 +31,7 @@ category: minutes * Nightly builds working * Nov: 14.2, point release with minor improvements and new build system -### Ideas +### Ideas * Quantum computing (IBM and Honeywell have free-tier APIs) XXXL * Python bindings for DSL (XXXL) * Automating or maintaining Docker images (XL, including docs) @@ -45,7 +45,7 @@ category: minutes * Look at OpenNLP for tokenizing and other processing * New embedding similarity job -### Summary and next steps +### Summary and next steps * DONE: Get travis migration done, nightlies running * TODO in Nov: Push 14.2 out * Reach out to formerly active project members diff --git a/website/_posts/2023-11-29-Meeting-Minutes.md b/website/_posts/2023-11-29-Meeting-Minutes.md index 11d73c8c06..87a9cfe117 100644 --- a/website/_posts/2023-11-29-Meeting-Minutes.md +++ b/website/_posts/2023-11-29-Meeting-Minutes.md @@ -29,14 +29,14 @@ category: minutes ### Release plans * Nov: 14.2, point release with minor improvements and new build system -### Active Ideas +### Active Ideas * Quantum computing * Python bindings for DSL -### Summary and next steps +### Summary and next steps * TODO in Dec: Push 14.2 out * TODO Dec 3: Pair work (Trevor/Andrew) on beta of `mahout-q` [Proof of concept of quantum front-end](https://issues.apache.org/jira/browse/MAHOUT-2166) * TODO in Q12024: Submit proposal to NSF [Pathways to Enable Open-Source Ecosystems (POSE)](https://new.nsf.gov/funding/opportunities/pathways-enable-open-source-ecosystems-pose) * Reach out to formerly active project members -* Reach out to quantum compute people +* Reach out to quantum compute people * JIRA ranching diff --git a/website/_posts/2024-01-31-Meeting-Minutes.md b/website/_posts/2024-01-31-Meeting-Minutes.md index 1c393d6494..bf6d0b5d3c 100644 --- a/website/_posts/2024-01-31-Meeting-Minutes.md +++ b/website/_posts/2024-01-31-Meeting-Minutes.md @@ -22,13 +22,13 @@ category: minutes ### Release plans * Not discussed but qumat is pushed to `main` and nightlies are building the website from that branch -### Active Ideas +### Active Ideas * Quantum computing * Python bindings for DSL -### Summary and next steps +### Summary and next steps * Completed work on beta of `qumat` [Proof of concept of quantum front-end](https://issues.apache.org/jira/browse/MAHOUT-2166) * Completed JIRA ranching * TODO in Q12024: Submit proposal to NSF [Pathways to Enable Open-Source Ecosystems (POSE)](https://new.nsf.gov/funding/opportunities/pathways-enable-open-source-ecosystems-pose) * Reach out to formerly active project members -* Reach out to quantum compute people +* Reach out to quantum compute people diff --git a/website/_posts/2024-02-14-Meeting-Minutes.md b/website/_posts/2024-02-14-Meeting-Minutes.md index 592934db48..c7bb0db3c3 100644 --- a/website/_posts/2024-02-14-Meeting-Minutes.md +++ b/website/_posts/2024-02-14-Meeting-Minutes.md @@ -29,4 +29,3 @@ category: minutes ### Other Business * JIRA hygiene with Shashanka in monthly Classic meeting - diff --git a/website/_posts/2024-03-07-Meeting-Minutes.md b/website/_posts/2024-03-07-Meeting-Minutes.md index ba0deda75a..7a3cd3c966 100644 --- a/website/_posts/2024-03-07-Meeting-Minutes.md +++ b/website/_posts/2024-03-07-Meeting-Minutes.md @@ -35,4 +35,3 @@ category: minutes 1. Ask INFRA to help us make sure PRs are defaulting to main instead of trunk (akm) 1. Make ticket to add notebooks to notebooks directory in source tree (https://issues.apache.org/jira/browse/MAHOUT-2198) 1. JIRA hygiene with Shashanka in monthly Classic meeting (on hold until Shashanka or Eric picks this up, meeting cancelled otherwise) - diff --git a/website/_posts/2024-04-24-Meeting-Minutes.md b/website/_posts/2024-04-24-Meeting-Minutes.md index b0e24a618f..3a6975667b 100644 --- a/website/_posts/2024-04-24-Meeting-Minutes.md +++ b/website/_posts/2024-04-24-Meeting-Minutes.md @@ -12,9 +12,9 @@ category: minutes * Trevor Grant -All old and new business was pushed to next meeting due to lack of quorum. +All old and new business was pushed to next meeting due to lack of quorum. -Tommy and Trevor jointly reviewed [PR #442](https://github.com/apache/mahout/pull/442). +Tommy and Trevor jointly reviewed [PR #442](https://github.com/apache/mahout/pull/442). The tl;dr is that it will be merged following some clean up on the git log. (Trevor walked Tommy through a chatGPT he made about how to clean it up). Don't forget the Community Happy Hour on Monday 4/29! (Ask for link on user@m.a.o or the #mahout channel on the-asf.slack.com, all are welcome!) diff --git a/website/_posts/2024-08-30-Meeting-Minutes.md b/website/_posts/2024-08-30-Meeting-Minutes.md index e6bddafa11..db7398bd27 100644 --- a/website/_posts/2024-08-30-Meeting-Minutes.md +++ b/website/_posts/2024-08-30-Meeting-Minutes.md @@ -44,4 +44,3 @@ category: minutes - [456](https://github.com/apache/mahout/issues/456) - [457](https://github.com/apache/mahout/issues/457) - [458](https://github.com/apache/mahout/issues/458) - diff --git a/website/_posts/2024-09-20-Meeting-Minutes.md b/website/_posts/2024-09-20-Meeting-Minutes.md index 1be8752bc0..05a242d2d4 100644 --- a/website/_posts/2024-09-20-Meeting-Minutes.md +++ b/website/_posts/2024-09-20-Meeting-Minutes.md @@ -41,4 +41,3 @@ category: minutes * https://github.com/apache/mahout/issues/457 * https://github.com/apache/mahout/issues/458 * https://github.com/apache/mahout/wiki/Data-Encoding-and-Kernel-Notes - diff --git a/website/_posts/2024-10-04-Meeting-Minutes.md b/website/_posts/2024-10-04-Meeting-Minutes.md index 0409f036c7..b044fbd02b 100644 --- a/website/_posts/2024-10-04-Meeting-Minutes.md +++ b/website/_posts/2024-10-04-Meeting-Minutes.md @@ -50,4 +50,3 @@ category: minutes * [#468 Add a Parameter object](https://github.com/apache/mahout/issues/468) * [#469 Bind parameter values at execution](https://github.com/apache/mahout/issues/469) * [#470 Add Qumat to PyPi](https://github.com/apache/mahout/issues/470) - diff --git a/website/_posts/2024-11-01-Meeting-Minutes.md b/website/_posts/2024-11-01-Meeting-Minutes.md index 555e97658e..5c53025ce6 100644 --- a/website/_posts/2024-11-01-Meeting-Minutes.md +++ b/website/_posts/2024-11-01-Meeting-Minutes.md @@ -45,4 +45,3 @@ Trevor and Andrew taking these * [#468 Add a Parameter object](https://github.com/apache/mahout/issues/468) * [#469 Bind parameter values at execution](https://github.com/apache/mahout/issues/469) * [#470 Add Qumat to PyPi](https://github.com/apache/mahout/issues/470) - diff --git a/website/_posts/2025-08-01-Meeting-Minutes.md b/website/_posts/2025-08-01-Meeting-Minutes.md index 0e67bed255..09bc03bd27 100644 --- a/website/_posts/2025-08-01-Meeting-Minutes.md +++ b/website/_posts/2025-08-01-Meeting-Minutes.md @@ -13,12 +13,12 @@ category: minutes * Krishna Dave ### Discussion -* Andrew and Trevor discussed certain UI changes that need to be implemented. -* They will be reviewing Krishna’s PRs for GSoC in the coming days. +* Andrew and Trevor discussed certain UI changes that need to be implemented. +* They will be reviewing Krishna’s PRs for GSoC in the coming days. * Krishna will create GitHub issues for all files that require updates, based on the provided sheet, to track progress and assign work accordingly. ### Next Steps -* Andrew and Trevor to review Krishna’s PRs. -* Krishna to prepare and log GitHub issues for required file updates. +* Andrew and Trevor to review Krishna’s PRs. +* Krishna to prepare and log GitHub issues for required file updates. * Follow up on UI changes discussed in the meeting. [Minutes](https://github.com/apache/mahout/discussions/551) on GitHub Discussions. diff --git a/website/_posts/2025-10-24-Meeting-Minutes.md b/website/_posts/2025-10-24-Meeting-Minutes.md index e84975ef0f..da4bf9cf7a 100644 --- a/website/_posts/2025-10-24-Meeting-Minutes.md +++ b/website/_posts/2025-10-24-Meeting-Minutes.md @@ -20,4 +20,4 @@ category: minutes * Krishna, Wes to learn more about PyPi and make accounts * New release 0.4.1 to be done, currently on 0.4. * Andrew to try using ATR for 0.4.1 release -[Minutes](https://github.com/apache/mahout/discussions/551) on GitHub Discussions. \ No newline at end of file +[Minutes](https://github.com/apache/mahout/discussions/551) on GitHub Discussions. diff --git a/website/about/how-to-contribute.md b/website/about/how-to-contribute.md index 875728b505..b0eda2a92a 100644 --- a/website/about/how-to-contribute.md +++ b/website/about/how-to-contribute.md @@ -19,7 +19,7 @@ Discussions at Apache happen on the mailing list. To get involved, you should jo to understand where the project is headed. * The **commit list** (to see changes as they are made) -Please keep discussions about Mahout on list so that everyone benefits. +Please keep discussions about Mahout on list so that everyone benefits. Emailing individual committers with questions about specific Mahout issues is discouraged. See [http://people.apache.org/~hossman/#private_q](http://people.apache.org/~hossman/#private_q) . Apache has a number of [email tips for contributors][1] as well. @@ -44,11 +44,11 @@ what we are working on in the internals and where you can get a sense of where people are working. Also, documentation is a great way to familiarize yourself with the code -and is always a welcome addition to the codebase and this website. Feel free -to contribute texts and tutorials! Committers will make sure they are added +and is always a welcome addition to the codebase and this website. Feel free +to contribute texts and tutorials! Committers will make sure they are added to this website, and we have a [guide for making website updates][2]. -We also have a [wide variety of books and slides][3] for learning more about -machine learning algorithms. +We also have a [wide variety of books and slides][3] for learning more about +machine learning algorithms. If you are interested in working towards being a committer, [general guidelines are available online](/documentation/developers/how-to-become-a-committer.html). @@ -64,7 +64,7 @@ against possible future changes). Please note that these are the "optimal" steps, and community members that don't have the time or resources to do everything outlined on this below should not be discouraged from submitting their ideas "as is" per "Yonik -Seeley's (Solr committer) Law of Patches": +Seeley's (Solr committer) Law of Patches": *A half-baked patch in Jira, with no documentation, no tests and no backwards compatibility is better than no patch at all.* @@ -76,7 +76,7 @@ and possibly improve it. ## Getting the source code -First of all, you need to get the [Mahout source code](/documentation/developers/version-control.html). Most development is done on the "trunk". Mahout mirrors its codebase on [GitHub](https://github.com/apache/mahout). The first step to making a contribution is to fork Mahout's master branch to your GitHub repository. +First of all, you need to get the [Mahout source code](/documentation/developers/version-control.html). Most development is done on the "trunk". Mahout mirrors its codebase on [GitHub](https://github.com/apache/mahout). The first step to making a contribution is to fork Mahout's master branch to your GitHub repository. @@ -86,24 +86,24 @@ Before you start, you should send a message to the [Mahout developer mailing lis (note: you have to subscribe before you can post), or file a ticket in our [issue tracker](/documentation/developers/issue-tracker.html). Describe your proposed changes and check that they fit in with what others are doing and have planned for the project. Be patient, it may take folks a while to understand your requirements. - 1. Create a JIRA Issue (if one does not already exist or you haven't already) - 2. Pull the code from your GitHub repository + 1. Create a JIRA Issue (if one does not already exist or you haven't already) + 2. Pull the code from your GitHub repository 3. Ensure that you are working with the latest code from the [apache/mahout](https://github.com/apache/mahout) master branch. - 3. Modify the source code and add some (very) nice features. + 3. Modify the source code and add some (very) nice features. - Be sure to adhere to the following points: - All public classes and methods should have informative Javadoc - comments. + comments. - Code should be formatted according to standard [Java coding conventions](http://www.oracle.com/technetwork/java/codeconventions-150003.pdf), with two exceptions: - - indent two spaces per level, not four. - - lines can be 120 characters, not 80. - - Contributions should pass existing unit tests. + - indent two spaces per level, not four. + - lines can be 120 characters, not 80. + - Contributions should pass existing unit tests. - New unit tests should be provided to demonstrate bugs and fixes. - 4. Commit the changes to your local repository. + 4. Commit the changes to your local repository. 4. Push the code back up to your GitHub repository. 5. Create a [Pull Request](https://help.github.com/articles/creating-a-pull-request) to the to apache/mahout repository on Github. - - Include the corresponding JIRA Issue number and description in the title of the pull request: + - Include the corresponding JIRA Issue number and description in the title of the pull request: - ie. MAHOUT-xxxx: < JIRA-Issue-Description > 6. Committers and other members of the Mahout community can then comment on the Pull Request. Be sure to watch for comments, respond and make any necessary changes. @@ -115,7 +115,7 @@ incorporate other's suggestions into into your changes if you think they're reas Please make sure that all unit tests succeed before creating your Pull Request. -Run *mvn clean test*, if you see *BUILD SUCCESSFUL* after the tests have finished, all is ok, but if you see *BUILD FAILED*, +Run *mvn clean test*, if you see *BUILD SUCCESSFUL* after the tests have finished, all is ok, but if you see *BUILD FAILED*, please carefully read the errors messages and check your code. #### Do's and Don'ts diff --git a/website/assets/asf_new_logo.svg b/website/assets/asf_new_logo.svg index f1084dfb36..830e886bc4 100644 --- a/website/assets/asf_new_logo.svg +++ b/website/assets/asf_new_logo.svg @@ -40,4 +40,4 @@ - \ No newline at end of file + diff --git a/website/assets/vendor/bootstrap/js/bootstrap.min.js b/website/assets/vendor/bootstrap/js/bootstrap.min.js index 80de23fa4d..6d98e10bb5 100755 --- a/website/assets/vendor/bootstrap/js/bootstrap.min.js +++ b/website/assets/vendor/bootstrap/js/bootstrap.min.js @@ -3,4 +3,4 @@ * Copyright 2011-2017 The Bootstrap Authors (https://github.com/twbs/bootstrap/graphs/contributors) * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) */ -if("undefined"==typeof jQuery)throw new Error("Bootstrap's JavaScript requires jQuery. jQuery must be included before Bootstrap's JavaScript.");!function(t){var e=t.fn.jquery.split(" ")[0].split(".");if(e[0]<2&&e[1]<9||1==e[0]&&9==e[1]&&e[2]<1||e[0]>=4)throw new Error("Bootstrap's JavaScript requires at least jQuery v1.9.1 but less than v4.0.0")}(jQuery),function(){function t(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function e(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}var i="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t},o=function(){function t(t,e){for(var n=0;n0?n:null}catch(t){return null}},reflow:function(t){return t.offsetHeight},triggerTransitionEnd:function(e){t(e).trigger(s.end)},supportsTransitionEnd:function(){return Boolean(s)},typeCheckConfig:function(t,i,o){for(var r in o)if(o.hasOwnProperty(r)){var s=o[r],a=i[r],l=a&&n(a)?"element":e(a);if(!new RegExp(s).test(l))throw new Error(t.toUpperCase()+': Option "'+r+'" provided type "'+l+'" but expected type "'+s+'".')}}};return s=o(),t.fn.emulateTransitionEnd=r,l.supportsTransitionEnd()&&(t.event.special[l.TRANSITION_END]=i()),l}(jQuery),s=(function(t){var e="alert",i=t.fn[e],s={DISMISS:'[data-dismiss="alert"]'},a={CLOSE:"close.bs.alert",CLOSED:"closed.bs.alert",CLICK_DATA_API:"click.bs.alert.data-api"},l={ALERT:"alert",FADE:"fade",SHOW:"show"},h=function(){function e(t){n(this,e),this._element=t}return e.prototype.close=function(t){t=t||this._element;var e=this._getRootElement(t);this._triggerCloseEvent(e).isDefaultPrevented()||this._removeElement(e)},e.prototype.dispose=function(){t.removeData(this._element,"bs.alert"),this._element=null},e.prototype._getRootElement=function(e){var n=r.getSelectorFromElement(e),i=!1;return n&&(i=t(n)[0]),i||(i=t(e).closest("."+l.ALERT)[0]),i},e.prototype._triggerCloseEvent=function(e){var n=t.Event(a.CLOSE);return t(e).trigger(n),n},e.prototype._removeElement=function(e){var n=this;t(e).removeClass(l.SHOW),r.supportsTransitionEnd()&&t(e).hasClass(l.FADE)?t(e).one(r.TRANSITION_END,function(t){return n._destroyElement(e,t)}).emulateTransitionEnd(150):this._destroyElement(e)},e.prototype._destroyElement=function(e){t(e).detach().trigger(a.CLOSED).remove()},e._jQueryInterface=function(n){return this.each(function(){var i=t(this),o=i.data("bs.alert");o||(o=new e(this),i.data("bs.alert",o)),"close"===n&&o[n](this)})},e._handleDismiss=function(t){return function(e){e&&e.preventDefault(),t.close(this)}},o(e,null,[{key:"VERSION",get:function(){return"4.0.0-alpha.6"}}]),e}();t(document).on(a.CLICK_DATA_API,s.DISMISS,h._handleDismiss(new h)),t.fn[e]=h._jQueryInterface,t.fn[e].Constructor=h,t.fn[e].noConflict=function(){return t.fn[e]=i,h._jQueryInterface}}(jQuery),function(t){var e="button",i=t.fn[e],r={ACTIVE:"active",BUTTON:"btn",FOCUS:"focus"},s={DATA_TOGGLE_CARROT:'[data-toggle^="button"]',DATA_TOGGLE:'[data-toggle="buttons"]',INPUT:"input",ACTIVE:".active",BUTTON:".btn"},a={CLICK_DATA_API:"click.bs.button.data-api",FOCUS_BLUR_DATA_API:"focus.bs.button.data-api blur.bs.button.data-api"},l=function(){function e(t){n(this,e),this._element=t}return e.prototype.toggle=function(){var e=!0,n=!0,i=t(this._element).closest(s.DATA_TOGGLE)[0];if(i){var o=t(this._element).find(s.INPUT)[0];if(o){if("radio"===o.type)if(o.checked&&t(this._element).hasClass(r.ACTIVE))e=!1;else{var a=t(i).find(s.ACTIVE)[0];a&&t(a).removeClass(r.ACTIVE)}if(e){if(o.hasAttribute("disabled")||i.hasAttribute("disabled")||o.classList.contains("disabled")||i.classList.contains("disabled"))return;o.checked=!t(this._element).hasClass(r.ACTIVE),t(o).trigger("change")}o.focus(),n=!1}}n&&this._element.setAttribute("aria-pressed",!t(this._element).hasClass(r.ACTIVE)),e&&t(this._element).toggleClass(r.ACTIVE)},e.prototype.dispose=function(){t.removeData(this._element,"bs.button"),this._element=null},e._jQueryInterface=function(n){return this.each(function(){var i=t(this).data("bs.button");i||(i=new e(this),t(this).data("bs.button",i)),"toggle"===n&&i[n]()})},o(e,null,[{key:"VERSION",get:function(){return"4.0.0-alpha.6"}}]),e}();t(document).on(a.CLICK_DATA_API,s.DATA_TOGGLE_CARROT,function(e){e.preventDefault();var n=e.target;t(n).hasClass(r.BUTTON)||(n=t(n).closest(s.BUTTON)),l._jQueryInterface.call(t(n),"toggle")}).on(a.FOCUS_BLUR_DATA_API,s.DATA_TOGGLE_CARROT,function(e){var n=t(e.target).closest(s.BUTTON)[0];t(n).toggleClass(r.FOCUS,/^focus(in)?$/.test(e.type))}),t.fn[e]=l._jQueryInterface,t.fn[e].Constructor=l,t.fn[e].noConflict=function(){return t.fn[e]=i,l._jQueryInterface}}(jQuery),function(t){var e="carousel",s="bs.carousel",a="."+s,l=t.fn[e],h={interval:5e3,keyboard:!0,slide:!1,pause:"hover",wrap:!0},c={interval:"(number|boolean)",keyboard:"boolean",slide:"(boolean|string)",pause:"(string|boolean)",wrap:"boolean"},u={NEXT:"next",PREV:"prev",LEFT:"left",RIGHT:"right"},d={SLIDE:"slide"+a,SLID:"slid"+a,KEYDOWN:"keydown"+a,MOUSEENTER:"mouseenter"+a,MOUSELEAVE:"mouseleave"+a,TOUCHEND:"touchend"+a,LOAD_DATA_API:"load.bs.carousel.data-api",CLICK_DATA_API:"click.bs.carousel.data-api"},f={CAROUSEL:"carousel",ACTIVE:"active",SLIDE:"slide",RIGHT:"carousel-item-right",LEFT:"carousel-item-left",NEXT:"carousel-item-next",PREV:"carousel-item-prev",ITEM:"carousel-item"},p={ACTIVE:".active",ACTIVE_ITEM:".active.carousel-item",ITEM:".carousel-item",NEXT_PREV:".carousel-item-next, .carousel-item-prev",INDICATORS:".carousel-indicators",DATA_SLIDE:"[data-slide], [data-slide-to]",DATA_RIDE:'[data-ride="carousel"]'},_=function(){function l(e,i){n(this,l),this._items=null,this._interval=null,this._activeElement=null,this._isPaused=!1,this._isSliding=!1,this.touchTimeout=null,this._config=this._getConfig(i),this._element=t(e)[0],this._indicatorsElement=t(this._element).find(p.INDICATORS)[0],this._addEventListeners()}return l.prototype.next=function(){this._isSliding||this._slide(u.NEXT)},l.prototype.nextWhenVisible=function(){document.hidden||this.next()},l.prototype.prev=function(){this._isSliding||this._slide(u.PREV)},l.prototype.pause=function(e){e||(this._isPaused=!0),t(this._element).find(p.NEXT_PREV)[0]&&r.supportsTransitionEnd()&&(r.triggerTransitionEnd(this._element),this.cycle(!0)),clearInterval(this._interval),this._interval=null},l.prototype.cycle=function(t){t||(this._isPaused=!1),this._interval&&(clearInterval(this._interval),this._interval=null),this._config.interval&&!this._isPaused&&(this._interval=setInterval((document.visibilityState?this.nextWhenVisible:this.next).bind(this),this._config.interval))},l.prototype.to=function(e){var n=this;this._activeElement=t(this._element).find(p.ACTIVE_ITEM)[0];var i=this._getItemIndex(this._activeElement);if(!(e>this._items.length-1||e<0))if(this._isSliding)t(this._element).one(d.SLID,function(){return n.to(e)});else{if(i===e)return this.pause(),void this.cycle();var o=e>i?u.NEXT:u.PREV;this._slide(o,this._items[e])}},l.prototype.dispose=function(){t(this._element).off(a),t.removeData(this._element,s),this._items=null,this._config=null,this._element=null,this._interval=null,this._isPaused=null,this._isSliding=null,this._activeElement=null,this._indicatorsElement=null},l.prototype._getConfig=function(n){return n=t.extend({},h,n),r.typeCheckConfig(e,n,c),n},l.prototype._addEventListeners=function(){var e=this;this._config.keyboard&&t(this._element).on(d.KEYDOWN,function(t){return e._keydown(t)}),"hover"===this._config.pause&&(t(this._element).on(d.MOUSEENTER,function(t){return e.pause(t)}).on(d.MOUSELEAVE,function(t){return e.cycle(t)}),"ontouchstart"in document.documentElement&&t(this._element).on(d.TOUCHEND,function(){e.pause(),e.touchTimeout&&clearTimeout(e.touchTimeout),e.touchTimeout=setTimeout(function(t){return e.cycle(t)},500+e._config.interval)}))},l.prototype._keydown=function(t){if(!/input|textarea/i.test(t.target.tagName))switch(t.which){case 37:t.preventDefault(),this.prev();break;case 39:t.preventDefault(),this.next();break;default:return}},l.prototype._getItemIndex=function(e){return this._items=t.makeArray(t(e).parent().find(p.ITEM)),this._items.indexOf(e)},l.prototype._getItemByDirection=function(t,e){var n=t===u.NEXT,i=t===u.PREV,o=this._getItemIndex(e),r=this._items.length-1;if((i&&0===o||n&&o===r)&&!this._config.wrap)return e;var s=(o+(t===u.PREV?-1:1))%this._items.length;return-1===s?this._items[this._items.length-1]:this._items[s]},l.prototype._triggerSlideEvent=function(e,n){var i=this._getItemIndex(e),o=this._getItemIndex(t(this._element).find(p.ACTIVE_ITEM)[0]),r=t.Event(d.SLIDE,{relatedTarget:e,direction:n,from:o,to:i});return t(this._element).trigger(r),r},l.prototype._setActiveIndicatorElement=function(e){if(this._indicatorsElement){t(this._indicatorsElement).find(p.ACTIVE).removeClass(f.ACTIVE);var n=this._indicatorsElement.children[this._getItemIndex(e)];n&&t(n).addClass(f.ACTIVE)}},l.prototype._slide=function(e,n){var i=this,o=t(this._element).find(p.ACTIVE_ITEM)[0],s=this._getItemIndex(o),a=n||o&&this._getItemByDirection(e,o),l=this._getItemIndex(a),h=Boolean(this._interval),c=void 0,_=void 0,g=void 0;if(e===u.NEXT?(c=f.LEFT,_=f.NEXT,g=u.LEFT):(c=f.RIGHT,_=f.PREV,g=u.RIGHT),a&&t(a).hasClass(f.ACTIVE))this._isSliding=!1;else if(!this._triggerSlideEvent(a,g).isDefaultPrevented()&&o&&a){this._isSliding=!0,h&&this.pause(),this._setActiveIndicatorElement(a);var m=t.Event(d.SLID,{relatedTarget:a,direction:g,from:s,to:l});r.supportsTransitionEnd()&&t(this._element).hasClass(f.SLIDE)?(t(a).addClass(_),r.reflow(a),t(o).addClass(c),t(a).addClass(c),t(o).one(r.TRANSITION_END,function(){t(a).removeClass(c+" "+_).addClass(f.ACTIVE),t(o).removeClass(f.ACTIVE+" "+_+" "+c),i._isSliding=!1,setTimeout(function(){return t(i._element).trigger(m)},0)}).emulateTransitionEnd(600)):(t(o).removeClass(f.ACTIVE),t(a).addClass(f.ACTIVE),this._isSliding=!1,t(this._element).trigger(m)),h&&this.cycle()}},l._jQueryInterface=function(e){return this.each(function(){var n=t(this).data(s),o=t.extend({},h,t(this).data());"object"===(void 0===e?"undefined":i(e))&&t.extend(o,e);var r="string"==typeof e?e:o.slide;if(n||(n=new l(this,o),t(this).data(s,n)),"number"==typeof e)n.to(e);else if("string"==typeof r){if(void 0===n[r])throw new Error('No method named "'+r+'"');n[r]()}else o.interval&&(n.pause(),n.cycle())})},l._dataApiClickHandler=function(e){var n=r.getSelectorFromElement(this);if(n){var i=t(n)[0];if(i&&t(i).hasClass(f.CAROUSEL)){var o=t.extend({},t(i).data(),t(this).data()),a=this.getAttribute("data-slide-to");a&&(o.interval=!1),l._jQueryInterface.call(t(i),o),a&&t(i).data(s).to(a),e.preventDefault()}}},o(l,null,[{key:"VERSION",get:function(){return"4.0.0-alpha.6"}},{key:"Default",get:function(){return h}}]),l}();t(document).on(d.CLICK_DATA_API,p.DATA_SLIDE,_._dataApiClickHandler),t(window).on(d.LOAD_DATA_API,function(){t(p.DATA_RIDE).each(function(){var e=t(this);_._jQueryInterface.call(e,e.data())})}),t.fn[e]=_._jQueryInterface,t.fn[e].Constructor=_,t.fn[e].noConflict=function(){return t.fn[e]=l,_._jQueryInterface}}(jQuery),function(t){var e="collapse",s="bs.collapse",a=t.fn[e],l={toggle:!0,parent:""},h={toggle:"boolean",parent:"string"},c={SHOW:"show.bs.collapse",SHOWN:"shown.bs.collapse",HIDE:"hide.bs.collapse",HIDDEN:"hidden.bs.collapse",CLICK_DATA_API:"click.bs.collapse.data-api"},u={SHOW:"show",COLLAPSE:"collapse",COLLAPSING:"collapsing",COLLAPSED:"collapsed"},d={WIDTH:"width",HEIGHT:"height"},f={ACTIVES:".show, .collapsing",DATA_TOGGLE:'[data-toggle="collapse"]'},p=function(){function a(e,i){n(this,a),this._isTransitioning=!1,this._element=e,this._config=this._getConfig(i),this._triggerArray=t.makeArray(t('[data-toggle="collapse"][href="#'+e.id+'"],[data-toggle="collapse"][data-target="#'+e.id+'"]'));for(var o=t(f.DATA_TOGGLE),s=0;s0&&this._triggerArray.push(l)}this._parent=this._config.parent?this._getParent():null,this._config.parent||this._addAriaAndCollapsedClass(this._element,this._triggerArray),this._config.toggle&&this.toggle()}return a.prototype.toggle=function(){t(this._element).hasClass(u.SHOW)?this.hide():this.show()},a.prototype.show=function(){var e=this;if(!this._isTransitioning&&!t(this._element).hasClass(u.SHOW)){var n=void 0,i=void 0;if(this._parent&&((n=t.makeArray(t(this._parent).children().children(f.ACTIVES))).length||(n=null)),!(n&&(i=t(n).data(s))&&i._isTransitioning)){var o=t.Event(c.SHOW);if(t(this._element).trigger(o),!o.isDefaultPrevented()){n&&(a._jQueryInterface.call(t(n),"hide"),i||t(n).data(s,null));var l=this._getDimension();t(this._element).removeClass(u.COLLAPSE).addClass(u.COLLAPSING),this._element.style[l]=0,this._triggerArray.length&&t(this._triggerArray).removeClass(u.COLLAPSED).attr("aria-expanded",!0),this.setTransitioning(!0);var h=function(){t(e._element).removeClass(u.COLLAPSING).addClass(u.COLLAPSE).addClass(u.SHOW),e._element.style[l]="",e.setTransitioning(!1),t(e._element).trigger(c.SHOWN)};if(r.supportsTransitionEnd()){var d="scroll"+(l[0].toUpperCase()+l.slice(1));t(this._element).one(r.TRANSITION_END,h).emulateTransitionEnd(600),this._element.style[l]=this._element[d]+"px"}else h()}}}},a.prototype.hide=function(){var e=this;if(!this._isTransitioning&&t(this._element).hasClass(u.SHOW)){var n=t.Event(c.HIDE);if(t(this._element).trigger(n),!n.isDefaultPrevented()){var i=this._getDimension();if(this._element.style[i]=this._element.getBoundingClientRect()[i]+"px",r.reflow(this._element),t(this._element).addClass(u.COLLAPSING).removeClass(u.COLLAPSE).removeClass(u.SHOW),this._triggerArray.length)for(var o=0;o0},l.prototype._getPopperConfig=function(){var t={placement:this._getPlacement(),modifiers:{offset:{offset:this._config.offset},flip:{enabled:this._config.flip}}};return this._inNavbar&&(t.modifiers.applyStyle={enabled:!this._inNavbar}),t},l._jQueryInterface=function(e){return this.each(function(){var n=t(this).data(s),o="object"===(void 0===e?"undefined":i(e))?e:null;if(n||(n=new l(this,o),t(this).data(s,n)),"string"==typeof e){if(void 0===n[e])throw new Error('No method named "'+e+'"');n[e]()}})},l._clearMenus=function(e){if(!e||3!==e.which&&("keyup"!==e.type||9===e.which))for(var n=t.makeArray(t(d.DATA_TOGGLE)),i=0;i0&&r--,40===e.which&&rdocument.documentElement.clientHeight;!this._isBodyOverflowing&&t&&(this._element.style.paddingLeft=this._scrollbarWidth+"px"),this._isBodyOverflowing&&!t&&(this._element.style.paddingRight=this._scrollbarWidth+"px")},a.prototype._resetAdjustments=function(){this._element.style.paddingLeft="",this._element.style.paddingRight=""},a.prototype._checkScrollbar=function(){this._isBodyOverflowing=document.body.clientWidth=n){var i=this._targets[this._targets.length-1];this._activeTarget!==i&&this._activate(i)}else{if(this._activeTarget&&t0)return this._activeTarget=null,void this._clear();for(var o=this._offsets.length;o--;)this._activeTarget!==this._targets[o]&&t>=this._offsets[o]&&(void 0===this._offsets[o+1]||t .dropdown-menu .active"},l=function(){function e(t){n(this,e),this._element=t}return e.prototype.show=function(){var e=this;if(!(this._element.parentNode&&this._element.parentNode.nodeType===Node.ELEMENT_NODE&&t(this._element).hasClass(s.ACTIVE)||t(this._element).hasClass(s.DISABLED))){var n=void 0,o=void 0,l=t(this._element).closest(a.NAV_LIST_GROUP)[0],h=r.getSelectorFromElement(this._element);l&&(o=t.makeArray(t(l).find(a.ACTIVE)),o=o[o.length-1]);var c=t.Event(i.HIDE,{relatedTarget:this._element}),u=t.Event(i.SHOW,{relatedTarget:o});if(o&&t(o).trigger(c),t(this._element).trigger(u),!u.isDefaultPrevented()&&!c.isDefaultPrevented()){h&&(n=t(h)[0]),this._activate(this._element,l);var d=function(){var n=t.Event(i.HIDDEN,{relatedTarget:e._element}),r=t.Event(i.SHOWN,{relatedTarget:o});t(o).trigger(n),t(e._element).trigger(r)};n?this._activate(n,n.parentNode,d):d()}}},e.prototype.dispose=function(){t.removeData(this._element,"bs.tab"),this._element=null},e.prototype._activate=function(e,n,i){var o=this,l=t(n).find(a.ACTIVE)[0],h=i&&r.supportsTransitionEnd()&&l&&t(l).hasClass(s.FADE),c=function(){return o._transitionComplete(e,l,h,i)};l&&h?t(l).one(r.TRANSITION_END,c).emulateTransitionEnd(150):c(),l&&t(l).removeClass(s.SHOW)},e.prototype._transitionComplete=function(e,n,i,o){if(n){t(n).removeClass(s.ACTIVE);var l=t(n.parentNode).find(a.DROPDOWN_ACTIVE_CHILD)[0];l&&t(l).removeClass(s.ACTIVE),n.setAttribute("aria-expanded",!1)}if(t(e).addClass(s.ACTIVE),e.setAttribute("aria-expanded",!0),i?(r.reflow(e),t(e).addClass(s.SHOW)):t(e).removeClass(s.FADE),e.parentNode&&t(e.parentNode).hasClass(s.DROPDOWN_MENU)){var h=t(e).closest(a.DROPDOWN)[0];h&&t(h).find(a.DROPDOWN_TOGGLE).addClass(s.ACTIVE),e.setAttribute("aria-expanded",!0)}o&&o()},e._jQueryInterface=function(n){return this.each(function(){var i=t(this),o=i.data("bs.tab");if(o||(o=new e(this),i.data("bs.tab",o)),"string"==typeof n){if(void 0===o[n])throw new Error('No method named "'+n+'"');o[n]()}})},o(e,null,[{key:"VERSION",get:function(){return"4.0.0-alpha.6"}}]),e}();t(document).on(i.CLICK_DATA_API,a.DATA_TOGGLE,function(e){e.preventDefault(),l._jQueryInterface.call(t(this),"show")}),t.fn.tab=l._jQueryInterface,t.fn.tab.Constructor=l,t.fn.tab.noConflict=function(){return t.fn.tab=e,l._jQueryInterface}}(jQuery),function(t){if("undefined"==typeof Popper)throw new Error("Bootstrap tooltips require Popper.js (https://popper.js.org)");var e="tooltip",s=".bs.tooltip",a=t.fn[e],l=new RegExp("(^|\\s)bs-tooltip\\S+","g"),h={animation:"boolean",template:"string",title:"(string|element|function)",trigger:"string",delay:"(number|object)",html:"boolean",selector:"(string|boolean)",placement:"(string|function)",offset:"(number|string)",container:"(string|element|boolean)",fallbackPlacement:"(string|array)"},c={AUTO:"auto",TOP:"top",RIGHT:"right",BOTTOM:"bottom",LEFT:"left"},u={animation:!0,template:'',trigger:"hover focus",title:"",delay:0,html:!1,selector:!1,placement:"top",offset:0,container:!1,fallbackPlacement:"flip"},d={SHOW:"show",OUT:"out"},f={HIDE:"hide"+s,HIDDEN:"hidden"+s,SHOW:"show"+s,SHOWN:"shown"+s,INSERTED:"inserted"+s,CLICK:"click"+s,FOCUSIN:"focusin"+s,FOCUSOUT:"focusout"+s,MOUSEENTER:"mouseenter"+s,MOUSELEAVE:"mouseleave"+s},p={FADE:"fade",SHOW:"show"},_={TOOLTIP:".tooltip",TOOLTIP_INNER:".tooltip-inner",ARROW:".arrow"},g={HOVER:"hover",FOCUS:"focus",CLICK:"click",MANUAL:"manual"},m=function(){function a(t,e){n(this,a),this._isEnabled=!0,this._timeout=0,this._hoverState="",this._activeTrigger={},this._popper=null,this.element=t,this.config=this._getConfig(e),this.tip=null,this._setListeners()}return a.prototype.enable=function(){this._isEnabled=!0},a.prototype.disable=function(){this._isEnabled=!1},a.prototype.toggleEnabled=function(){this._isEnabled=!this._isEnabled},a.prototype.toggle=function(e){if(e){var n=this.constructor.DATA_KEY,i=t(e.currentTarget).data(n);i||(i=new this.constructor(e.currentTarget,this._getDelegateConfig()),t(e.currentTarget).data(n,i)),i._activeTrigger.click=!i._activeTrigger.click,i._isWithActiveTrigger()?i._enter(null,i):i._leave(null,i)}else{if(t(this.getTipElement()).hasClass(p.SHOW))return void this._leave(null,this);this._enter(null,this)}},a.prototype.dispose=function(){clearTimeout(this._timeout),t.removeData(this.element,this.constructor.DATA_KEY),t(this.element).off(this.constructor.EVENT_KEY),t(this.element).closest(".modal").off("hide.bs.modal"),this.tip&&t(this.tip).remove(),this._isEnabled=null,this._timeout=null,this._hoverState=null,this._activeTrigger=null,null!==this._popper&&this._popper.destroy(),this._popper=null,this.element=null,this.config=null,this.tip=null},a.prototype.show=function(){var e=this;if("none"===t(this.element).css("display"))throw new Error("Please use show on visible elements");var n=t.Event(this.constructor.Event.SHOW);if(this.isWithContent()&&this._isEnabled){t(this.element).trigger(n);var i=t.contains(this.element.ownerDocument.documentElement,this.element);if(n.isDefaultPrevented()||!i)return;var o=this.getTipElement(),s=r.getUID(this.constructor.NAME);o.setAttribute("id",s),this.element.setAttribute("aria-describedby",s),this.setContent(),this.config.animation&&t(o).addClass(p.FADE);var l="function"==typeof this.config.placement?this.config.placement.call(this,o,this.element):this.config.placement,h=this._getAttachment(l);this.addAttachmentClass(h);var c=!1===this.config.container?document.body:t(this.config.container);t(o).data(this.constructor.DATA_KEY,this),t.contains(this.element.ownerDocument.documentElement,this.tip)||t(o).appendTo(c),t(this.element).trigger(this.constructor.Event.INSERTED),this._popper=new Popper(this.element,o,{placement:h,modifiers:{offset:{offset:this.config.offset},flip:{behavior:this.config.fallbackPlacement},arrow:{element:_.ARROW}},onCreate:function(t){t.originalPlacement!==t.placement&&e._handlePopperPlacementChange(t)},onUpdate:function(t){e._handlePopperPlacementChange(t)}}),t(o).addClass(p.SHOW),"ontouchstart"in document.documentElement&&t("body").children().on("mouseover",null,t.noop);var u=function(){e.config.animation&&e._fixTransition();var n=e._hoverState;e._hoverState=null,t(e.element).trigger(e.constructor.Event.SHOWN),n===d.OUT&&e._leave(null,e)};r.supportsTransitionEnd()&&t(this.tip).hasClass(p.FADE)?t(this.tip).one(r.TRANSITION_END,u).emulateTransitionEnd(a._TRANSITION_DURATION):u()}},a.prototype.hide=function(e){var n=this,i=this.getTipElement(),o=t.Event(this.constructor.Event.HIDE),s=function(){n._hoverState!==d.SHOW&&i.parentNode&&i.parentNode.removeChild(i),n._cleanTipClass(),n.element.removeAttribute("aria-describedby"),t(n.element).trigger(n.constructor.Event.HIDDEN),null!==n._popper&&n._popper.destroy(),e&&e()};t(this.element).trigger(o),o.isDefaultPrevented()||(t(i).removeClass(p.SHOW),"ontouchstart"in document.documentElement&&t("body").children().off("mouseover",null,t.noop),this._activeTrigger[g.CLICK]=!1,this._activeTrigger[g.FOCUS]=!1,this._activeTrigger[g.HOVER]=!1,r.supportsTransitionEnd()&&t(this.tip).hasClass(p.FADE)?t(i).one(r.TRANSITION_END,s).emulateTransitionEnd(150):s(),this._hoverState="")},a.prototype.update=function(){null!==this._popper&&this._popper.scheduleUpdate()},a.prototype.isWithContent=function(){return Boolean(this.getTitle())},a.prototype.addAttachmentClass=function(e){t(this.getTipElement()).addClass("bs-tooltip-"+e)},a.prototype.getTipElement=function(){return this.tip=this.tip||t(this.config.template)[0]},a.prototype.setContent=function(){var e=t(this.getTipElement());this.setElementContent(e.find(_.TOOLTIP_INNER),this.getTitle()),e.removeClass(p.FADE+" "+p.SHOW)},a.prototype.setElementContent=function(e,n){var o=this.config.html;"object"===(void 0===n?"undefined":i(n))&&(n.nodeType||n.jquery)?o?t(n).parent().is(e)||e.empty().append(n):e.text(t(n).text()):e[o?"html":"text"](n)},a.prototype.getTitle=function(){var t=this.element.getAttribute("data-original-title");return t||(t="function"==typeof this.config.title?this.config.title.call(this.element):this.config.title),t},a.prototype._getAttachment=function(t){return c[t.toUpperCase()]},a.prototype._setListeners=function(){var e=this;this.config.trigger.split(" ").forEach(function(n){if("click"===n)t(e.element).on(e.constructor.Event.CLICK,e.config.selector,function(t){return e.toggle(t)});else if(n!==g.MANUAL){var i=n===g.HOVER?e.constructor.Event.MOUSEENTER:e.constructor.Event.FOCUSIN,o=n===g.HOVER?e.constructor.Event.MOUSELEAVE:e.constructor.Event.FOCUSOUT;t(e.element).on(i,e.config.selector,function(t){return e._enter(t)}).on(o,e.config.selector,function(t){return e._leave(t)})}t(e.element).closest(".modal").on("hide.bs.modal",function(){return e.hide()})}),this.config.selector?this.config=t.extend({},this.config,{trigger:"manual",selector:""}):this._fixTitle()},a.prototype._fixTitle=function(){var t=i(this.element.getAttribute("data-original-title"));(this.element.getAttribute("title")||"string"!==t)&&(this.element.setAttribute("data-original-title",this.element.getAttribute("title")||""),this.element.setAttribute("title",""))},a.prototype._enter=function(e,n){var i=this.constructor.DATA_KEY;(n=n||t(e.currentTarget).data(i))||(n=new this.constructor(e.currentTarget,this._getDelegateConfig()),t(e.currentTarget).data(i,n)),e&&(n._activeTrigger["focusin"===e.type?g.FOCUS:g.HOVER]=!0),t(n.getTipElement()).hasClass(p.SHOW)||n._hoverState===d.SHOW?n._hoverState=d.SHOW:(clearTimeout(n._timeout),n._hoverState=d.SHOW,n.config.delay&&n.config.delay.show?n._timeout=setTimeout(function(){n._hoverState===d.SHOW&&n.show()},n.config.delay.show):n.show())},a.prototype._leave=function(e,n){var i=this.constructor.DATA_KEY;(n=n||t(e.currentTarget).data(i))||(n=new this.constructor(e.currentTarget,this._getDelegateConfig()),t(e.currentTarget).data(i,n)),e&&(n._activeTrigger["focusout"===e.type?g.FOCUS:g.HOVER]=!1),n._isWithActiveTrigger()||(clearTimeout(n._timeout),n._hoverState=d.OUT,n.config.delay&&n.config.delay.hide?n._timeout=setTimeout(function(){n._hoverState===d.OUT&&n.hide()},n.config.delay.hide):n.hide())},a.prototype._isWithActiveTrigger=function(){for(var t in this._activeTrigger)if(this._activeTrigger[t])return!0;return!1},a.prototype._getConfig=function(n){return(n=t.extend({},this.constructor.Default,t(this.element).data(),n)).delay&&"number"==typeof n.delay&&(n.delay={show:n.delay,hide:n.delay}),n.title&&"number"==typeof n.title&&(n.title=n.title.toString()),n.content&&"number"==typeof n.content&&(n.content=n.content.toString()),r.typeCheckConfig(e,n,this.constructor.DefaultType),n},a.prototype._getDelegateConfig=function(){var t={};if(this.config)for(var e in this.config)this.constructor.Default[e]!==this.config[e]&&(t[e]=this.config[e]);return t},a.prototype._cleanTipClass=function(){var e=t(this.getTipElement()),n=e.attr("class").match(l);null!==n&&n.length>0&&e.removeClass(n.join(""))},a.prototype._handlePopperPlacementChange=function(t){this._cleanTipClass(),this.addAttachmentClass(this._getAttachment(t.placement))},a.prototype._fixTransition=function(){var e=this.getTipElement(),n=this.config.animation;null===e.getAttribute("x-placement")&&(t(e).removeClass(p.FADE),this.config.animation=!1,this.hide(),this.show(),this.config.animation=n)},a._jQueryInterface=function(e){return this.each(function(){var n=t(this).data("bs.tooltip"),o="object"===(void 0===e?"undefined":i(e))&&e;if((n||!/dispose|hide/.test(e))&&(n||(n=new a(this,o),t(this).data("bs.tooltip",n)),"string"==typeof e)){if(void 0===n[e])throw new Error('No method named "'+e+'"');n[e]()}})},o(a,null,[{key:"VERSION",get:function(){return"4.0.0-alpha.6"}},{key:"Default",get:function(){return u}},{key:"NAME",get:function(){return e}},{key:"DATA_KEY",get:function(){return"bs.tooltip"}},{key:"Event",get:function(){return f}},{key:"EVENT_KEY",get:function(){return s}},{key:"DefaultType",get:function(){return h}}]),a}();return t.fn[e]=m._jQueryInterface,t.fn[e].Constructor=m,t.fn[e].noConflict=function(){return t.fn[e]=a,m._jQueryInterface},m}(jQuery));!function(r){var a="popover",l=".bs.popover",h=r.fn[a],c=new RegExp("(^|\\s)bs-popover\\S+","g"),u=r.extend({},s.Default,{placement:"right",trigger:"click",content:"",template:''}),d=r.extend({},s.DefaultType,{content:"(string|element|function)"}),f={FADE:"fade",SHOW:"show"},p={TITLE:".popover-header",CONTENT:".popover-body"},_={HIDE:"hide"+l,HIDDEN:"hidden"+l,SHOW:"show"+l,SHOWN:"shown"+l,INSERTED:"inserted"+l,CLICK:"click"+l,FOCUSIN:"focusin"+l,FOCUSOUT:"focusout"+l,MOUSEENTER:"mouseenter"+l,MOUSELEAVE:"mouseleave"+l},g=function(s){function h(){return n(this,h),t(this,s.apply(this,arguments))}return e(h,s),h.prototype.isWithContent=function(){return this.getTitle()||this._getContent()},h.prototype.addAttachmentClass=function(t){r(this.getTipElement()).addClass("bs-popover-"+t)},h.prototype.getTipElement=function(){return this.tip=this.tip||r(this.config.template)[0]},h.prototype.setContent=function(){var t=r(this.getTipElement());this.setElementContent(t.find(p.TITLE),this.getTitle()),this.setElementContent(t.find(p.CONTENT),this._getContent()),t.removeClass(f.FADE+" "+f.SHOW)},h.prototype._getContent=function(){return this.element.getAttribute("data-content")||("function"==typeof this.config.content?this.config.content.call(this.element):this.config.content)},h.prototype._cleanTipClass=function(){var t=r(this.getTipElement()),e=t.attr("class").match(c);null!==e&&e.length>0&&t.removeClass(e.join(""))},h._jQueryInterface=function(t){return this.each(function(){var e=r(this).data("bs.popover"),n="object"===(void 0===t?"undefined":i(t))?t:null;if((e||!/destroy|hide/.test(t))&&(e||(e=new h(this,n),r(this).data("bs.popover",e)),"string"==typeof t)){if(void 0===e[t])throw new Error('No method named "'+t+'"');e[t]()}})},o(h,null,[{key:"VERSION",get:function(){return"4.0.0-alpha.6"}},{key:"Default",get:function(){return u}},{key:"NAME",get:function(){return a}},{key:"DATA_KEY",get:function(){return"bs.popover"}},{key:"Event",get:function(){return _}},{key:"EVENT_KEY",get:function(){return l}},{key:"DefaultType",get:function(){return d}}]),h}(s);r.fn[a]=g._jQueryInterface,r.fn[a].Constructor=g,r.fn[a].noConflict=function(){return r.fn[a]=h,g._jQueryInterface}}(jQuery)}(); \ No newline at end of file +if("undefined"==typeof jQuery)throw new Error("Bootstrap's JavaScript requires jQuery. jQuery must be included before Bootstrap's JavaScript.");!function(t){var e=t.fn.jquery.split(" ")[0].split(".");if(e[0]<2&&e[1]<9||1==e[0]&&9==e[1]&&e[2]<1||e[0]>=4)throw new Error("Bootstrap's JavaScript requires at least jQuery v1.9.1 but less than v4.0.0")}(jQuery),function(){function t(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function e(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}var i="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t},o=function(){function t(t,e){for(var n=0;n0?n:null}catch(t){return null}},reflow:function(t){return t.offsetHeight},triggerTransitionEnd:function(e){t(e).trigger(s.end)},supportsTransitionEnd:function(){return Boolean(s)},typeCheckConfig:function(t,i,o){for(var r in o)if(o.hasOwnProperty(r)){var s=o[r],a=i[r],l=a&&n(a)?"element":e(a);if(!new RegExp(s).test(l))throw new Error(t.toUpperCase()+': Option "'+r+'" provided type "'+l+'" but expected type "'+s+'".')}}};return s=o(),t.fn.emulateTransitionEnd=r,l.supportsTransitionEnd()&&(t.event.special[l.TRANSITION_END]=i()),l}(jQuery),s=(function(t){var e="alert",i=t.fn[e],s={DISMISS:'[data-dismiss="alert"]'},a={CLOSE:"close.bs.alert",CLOSED:"closed.bs.alert",CLICK_DATA_API:"click.bs.alert.data-api"},l={ALERT:"alert",FADE:"fade",SHOW:"show"},h=function(){function e(t){n(this,e),this._element=t}return e.prototype.close=function(t){t=t||this._element;var e=this._getRootElement(t);this._triggerCloseEvent(e).isDefaultPrevented()||this._removeElement(e)},e.prototype.dispose=function(){t.removeData(this._element,"bs.alert"),this._element=null},e.prototype._getRootElement=function(e){var n=r.getSelectorFromElement(e),i=!1;return n&&(i=t(n)[0]),i||(i=t(e).closest("."+l.ALERT)[0]),i},e.prototype._triggerCloseEvent=function(e){var n=t.Event(a.CLOSE);return t(e).trigger(n),n},e.prototype._removeElement=function(e){var n=this;t(e).removeClass(l.SHOW),r.supportsTransitionEnd()&&t(e).hasClass(l.FADE)?t(e).one(r.TRANSITION_END,function(t){return n._destroyElement(e,t)}).emulateTransitionEnd(150):this._destroyElement(e)},e.prototype._destroyElement=function(e){t(e).detach().trigger(a.CLOSED).remove()},e._jQueryInterface=function(n){return this.each(function(){var i=t(this),o=i.data("bs.alert");o||(o=new e(this),i.data("bs.alert",o)),"close"===n&&o[n](this)})},e._handleDismiss=function(t){return function(e){e&&e.preventDefault(),t.close(this)}},o(e,null,[{key:"VERSION",get:function(){return"4.0.0-alpha.6"}}]),e}();t(document).on(a.CLICK_DATA_API,s.DISMISS,h._handleDismiss(new h)),t.fn[e]=h._jQueryInterface,t.fn[e].Constructor=h,t.fn[e].noConflict=function(){return t.fn[e]=i,h._jQueryInterface}}(jQuery),function(t){var e="button",i=t.fn[e],r={ACTIVE:"active",BUTTON:"btn",FOCUS:"focus"},s={DATA_TOGGLE_CARROT:'[data-toggle^="button"]',DATA_TOGGLE:'[data-toggle="buttons"]',INPUT:"input",ACTIVE:".active",BUTTON:".btn"},a={CLICK_DATA_API:"click.bs.button.data-api",FOCUS_BLUR_DATA_API:"focus.bs.button.data-api blur.bs.button.data-api"},l=function(){function e(t){n(this,e),this._element=t}return e.prototype.toggle=function(){var e=!0,n=!0,i=t(this._element).closest(s.DATA_TOGGLE)[0];if(i){var o=t(this._element).find(s.INPUT)[0];if(o){if("radio"===o.type)if(o.checked&&t(this._element).hasClass(r.ACTIVE))e=!1;else{var a=t(i).find(s.ACTIVE)[0];a&&t(a).removeClass(r.ACTIVE)}if(e){if(o.hasAttribute("disabled")||i.hasAttribute("disabled")||o.classList.contains("disabled")||i.classList.contains("disabled"))return;o.checked=!t(this._element).hasClass(r.ACTIVE),t(o).trigger("change")}o.focus(),n=!1}}n&&this._element.setAttribute("aria-pressed",!t(this._element).hasClass(r.ACTIVE)),e&&t(this._element).toggleClass(r.ACTIVE)},e.prototype.dispose=function(){t.removeData(this._element,"bs.button"),this._element=null},e._jQueryInterface=function(n){return this.each(function(){var i=t(this).data("bs.button");i||(i=new e(this),t(this).data("bs.button",i)),"toggle"===n&&i[n]()})},o(e,null,[{key:"VERSION",get:function(){return"4.0.0-alpha.6"}}]),e}();t(document).on(a.CLICK_DATA_API,s.DATA_TOGGLE_CARROT,function(e){e.preventDefault();var n=e.target;t(n).hasClass(r.BUTTON)||(n=t(n).closest(s.BUTTON)),l._jQueryInterface.call(t(n),"toggle")}).on(a.FOCUS_BLUR_DATA_API,s.DATA_TOGGLE_CARROT,function(e){var n=t(e.target).closest(s.BUTTON)[0];t(n).toggleClass(r.FOCUS,/^focus(in)?$/.test(e.type))}),t.fn[e]=l._jQueryInterface,t.fn[e].Constructor=l,t.fn[e].noConflict=function(){return t.fn[e]=i,l._jQueryInterface}}(jQuery),function(t){var e="carousel",s="bs.carousel",a="."+s,l=t.fn[e],h={interval:5e3,keyboard:!0,slide:!1,pause:"hover",wrap:!0},c={interval:"(number|boolean)",keyboard:"boolean",slide:"(boolean|string)",pause:"(string|boolean)",wrap:"boolean"},u={NEXT:"next",PREV:"prev",LEFT:"left",RIGHT:"right"},d={SLIDE:"slide"+a,SLID:"slid"+a,KEYDOWN:"keydown"+a,MOUSEENTER:"mouseenter"+a,MOUSELEAVE:"mouseleave"+a,TOUCHEND:"touchend"+a,LOAD_DATA_API:"load.bs.carousel.data-api",CLICK_DATA_API:"click.bs.carousel.data-api"},f={CAROUSEL:"carousel",ACTIVE:"active",SLIDE:"slide",RIGHT:"carousel-item-right",LEFT:"carousel-item-left",NEXT:"carousel-item-next",PREV:"carousel-item-prev",ITEM:"carousel-item"},p={ACTIVE:".active",ACTIVE_ITEM:".active.carousel-item",ITEM:".carousel-item",NEXT_PREV:".carousel-item-next, .carousel-item-prev",INDICATORS:".carousel-indicators",DATA_SLIDE:"[data-slide], [data-slide-to]",DATA_RIDE:'[data-ride="carousel"]'},_=function(){function l(e,i){n(this,l),this._items=null,this._interval=null,this._activeElement=null,this._isPaused=!1,this._isSliding=!1,this.touchTimeout=null,this._config=this._getConfig(i),this._element=t(e)[0],this._indicatorsElement=t(this._element).find(p.INDICATORS)[0],this._addEventListeners()}return l.prototype.next=function(){this._isSliding||this._slide(u.NEXT)},l.prototype.nextWhenVisible=function(){document.hidden||this.next()},l.prototype.prev=function(){this._isSliding||this._slide(u.PREV)},l.prototype.pause=function(e){e||(this._isPaused=!0),t(this._element).find(p.NEXT_PREV)[0]&&r.supportsTransitionEnd()&&(r.triggerTransitionEnd(this._element),this.cycle(!0)),clearInterval(this._interval),this._interval=null},l.prototype.cycle=function(t){t||(this._isPaused=!1),this._interval&&(clearInterval(this._interval),this._interval=null),this._config.interval&&!this._isPaused&&(this._interval=setInterval((document.visibilityState?this.nextWhenVisible:this.next).bind(this),this._config.interval))},l.prototype.to=function(e){var n=this;this._activeElement=t(this._element).find(p.ACTIVE_ITEM)[0];var i=this._getItemIndex(this._activeElement);if(!(e>this._items.length-1||e<0))if(this._isSliding)t(this._element).one(d.SLID,function(){return n.to(e)});else{if(i===e)return this.pause(),void this.cycle();var o=e>i?u.NEXT:u.PREV;this._slide(o,this._items[e])}},l.prototype.dispose=function(){t(this._element).off(a),t.removeData(this._element,s),this._items=null,this._config=null,this._element=null,this._interval=null,this._isPaused=null,this._isSliding=null,this._activeElement=null,this._indicatorsElement=null},l.prototype._getConfig=function(n){return n=t.extend({},h,n),r.typeCheckConfig(e,n,c),n},l.prototype._addEventListeners=function(){var e=this;this._config.keyboard&&t(this._element).on(d.KEYDOWN,function(t){return e._keydown(t)}),"hover"===this._config.pause&&(t(this._element).on(d.MOUSEENTER,function(t){return e.pause(t)}).on(d.MOUSELEAVE,function(t){return e.cycle(t)}),"ontouchstart"in document.documentElement&&t(this._element).on(d.TOUCHEND,function(){e.pause(),e.touchTimeout&&clearTimeout(e.touchTimeout),e.touchTimeout=setTimeout(function(t){return e.cycle(t)},500+e._config.interval)}))},l.prototype._keydown=function(t){if(!/input|textarea/i.test(t.target.tagName))switch(t.which){case 37:t.preventDefault(),this.prev();break;case 39:t.preventDefault(),this.next();break;default:return}},l.prototype._getItemIndex=function(e){return this._items=t.makeArray(t(e).parent().find(p.ITEM)),this._items.indexOf(e)},l.prototype._getItemByDirection=function(t,e){var n=t===u.NEXT,i=t===u.PREV,o=this._getItemIndex(e),r=this._items.length-1;if((i&&0===o||n&&o===r)&&!this._config.wrap)return e;var s=(o+(t===u.PREV?-1:1))%this._items.length;return-1===s?this._items[this._items.length-1]:this._items[s]},l.prototype._triggerSlideEvent=function(e,n){var i=this._getItemIndex(e),o=this._getItemIndex(t(this._element).find(p.ACTIVE_ITEM)[0]),r=t.Event(d.SLIDE,{relatedTarget:e,direction:n,from:o,to:i});return t(this._element).trigger(r),r},l.prototype._setActiveIndicatorElement=function(e){if(this._indicatorsElement){t(this._indicatorsElement).find(p.ACTIVE).removeClass(f.ACTIVE);var n=this._indicatorsElement.children[this._getItemIndex(e)];n&&t(n).addClass(f.ACTIVE)}},l.prototype._slide=function(e,n){var i=this,o=t(this._element).find(p.ACTIVE_ITEM)[0],s=this._getItemIndex(o),a=n||o&&this._getItemByDirection(e,o),l=this._getItemIndex(a),h=Boolean(this._interval),c=void 0,_=void 0,g=void 0;if(e===u.NEXT?(c=f.LEFT,_=f.NEXT,g=u.LEFT):(c=f.RIGHT,_=f.PREV,g=u.RIGHT),a&&t(a).hasClass(f.ACTIVE))this._isSliding=!1;else if(!this._triggerSlideEvent(a,g).isDefaultPrevented()&&o&&a){this._isSliding=!0,h&&this.pause(),this._setActiveIndicatorElement(a);var m=t.Event(d.SLID,{relatedTarget:a,direction:g,from:s,to:l});r.supportsTransitionEnd()&&t(this._element).hasClass(f.SLIDE)?(t(a).addClass(_),r.reflow(a),t(o).addClass(c),t(a).addClass(c),t(o).one(r.TRANSITION_END,function(){t(a).removeClass(c+" "+_).addClass(f.ACTIVE),t(o).removeClass(f.ACTIVE+" "+_+" "+c),i._isSliding=!1,setTimeout(function(){return t(i._element).trigger(m)},0)}).emulateTransitionEnd(600)):(t(o).removeClass(f.ACTIVE),t(a).addClass(f.ACTIVE),this._isSliding=!1,t(this._element).trigger(m)),h&&this.cycle()}},l._jQueryInterface=function(e){return this.each(function(){var n=t(this).data(s),o=t.extend({},h,t(this).data());"object"===(void 0===e?"undefined":i(e))&&t.extend(o,e);var r="string"==typeof e?e:o.slide;if(n||(n=new l(this,o),t(this).data(s,n)),"number"==typeof e)n.to(e);else if("string"==typeof r){if(void 0===n[r])throw new Error('No method named "'+r+'"');n[r]()}else o.interval&&(n.pause(),n.cycle())})},l._dataApiClickHandler=function(e){var n=r.getSelectorFromElement(this);if(n){var i=t(n)[0];if(i&&t(i).hasClass(f.CAROUSEL)){var o=t.extend({},t(i).data(),t(this).data()),a=this.getAttribute("data-slide-to");a&&(o.interval=!1),l._jQueryInterface.call(t(i),o),a&&t(i).data(s).to(a),e.preventDefault()}}},o(l,null,[{key:"VERSION",get:function(){return"4.0.0-alpha.6"}},{key:"Default",get:function(){return h}}]),l}();t(document).on(d.CLICK_DATA_API,p.DATA_SLIDE,_._dataApiClickHandler),t(window).on(d.LOAD_DATA_API,function(){t(p.DATA_RIDE).each(function(){var e=t(this);_._jQueryInterface.call(e,e.data())})}),t.fn[e]=_._jQueryInterface,t.fn[e].Constructor=_,t.fn[e].noConflict=function(){return t.fn[e]=l,_._jQueryInterface}}(jQuery),function(t){var e="collapse",s="bs.collapse",a=t.fn[e],l={toggle:!0,parent:""},h={toggle:"boolean",parent:"string"},c={SHOW:"show.bs.collapse",SHOWN:"shown.bs.collapse",HIDE:"hide.bs.collapse",HIDDEN:"hidden.bs.collapse",CLICK_DATA_API:"click.bs.collapse.data-api"},u={SHOW:"show",COLLAPSE:"collapse",COLLAPSING:"collapsing",COLLAPSED:"collapsed"},d={WIDTH:"width",HEIGHT:"height"},f={ACTIVES:".show, .collapsing",DATA_TOGGLE:'[data-toggle="collapse"]'},p=function(){function a(e,i){n(this,a),this._isTransitioning=!1,this._element=e,this._config=this._getConfig(i),this._triggerArray=t.makeArray(t('[data-toggle="collapse"][href="#'+e.id+'"],[data-toggle="collapse"][data-target="#'+e.id+'"]'));for(var o=t(f.DATA_TOGGLE),s=0;s0&&this._triggerArray.push(l)}this._parent=this._config.parent?this._getParent():null,this._config.parent||this._addAriaAndCollapsedClass(this._element,this._triggerArray),this._config.toggle&&this.toggle()}return a.prototype.toggle=function(){t(this._element).hasClass(u.SHOW)?this.hide():this.show()},a.prototype.show=function(){var e=this;if(!this._isTransitioning&&!t(this._element).hasClass(u.SHOW)){var n=void 0,i=void 0;if(this._parent&&((n=t.makeArray(t(this._parent).children().children(f.ACTIVES))).length||(n=null)),!(n&&(i=t(n).data(s))&&i._isTransitioning)){var o=t.Event(c.SHOW);if(t(this._element).trigger(o),!o.isDefaultPrevented()){n&&(a._jQueryInterface.call(t(n),"hide"),i||t(n).data(s,null));var l=this._getDimension();t(this._element).removeClass(u.COLLAPSE).addClass(u.COLLAPSING),this._element.style[l]=0,this._triggerArray.length&&t(this._triggerArray).removeClass(u.COLLAPSED).attr("aria-expanded",!0),this.setTransitioning(!0);var h=function(){t(e._element).removeClass(u.COLLAPSING).addClass(u.COLLAPSE).addClass(u.SHOW),e._element.style[l]="",e.setTransitioning(!1),t(e._element).trigger(c.SHOWN)};if(r.supportsTransitionEnd()){var d="scroll"+(l[0].toUpperCase()+l.slice(1));t(this._element).one(r.TRANSITION_END,h).emulateTransitionEnd(600),this._element.style[l]=this._element[d]+"px"}else h()}}}},a.prototype.hide=function(){var e=this;if(!this._isTransitioning&&t(this._element).hasClass(u.SHOW)){var n=t.Event(c.HIDE);if(t(this._element).trigger(n),!n.isDefaultPrevented()){var i=this._getDimension();if(this._element.style[i]=this._element.getBoundingClientRect()[i]+"px",r.reflow(this._element),t(this._element).addClass(u.COLLAPSING).removeClass(u.COLLAPSE).removeClass(u.SHOW),this._triggerArray.length)for(var o=0;o0},l.prototype._getPopperConfig=function(){var t={placement:this._getPlacement(),modifiers:{offset:{offset:this._config.offset},flip:{enabled:this._config.flip}}};return this._inNavbar&&(t.modifiers.applyStyle={enabled:!this._inNavbar}),t},l._jQueryInterface=function(e){return this.each(function(){var n=t(this).data(s),o="object"===(void 0===e?"undefined":i(e))?e:null;if(n||(n=new l(this,o),t(this).data(s,n)),"string"==typeof e){if(void 0===n[e])throw new Error('No method named "'+e+'"');n[e]()}})},l._clearMenus=function(e){if(!e||3!==e.which&&("keyup"!==e.type||9===e.which))for(var n=t.makeArray(t(d.DATA_TOGGLE)),i=0;i0&&r--,40===e.which&&rdocument.documentElement.clientHeight;!this._isBodyOverflowing&&t&&(this._element.style.paddingLeft=this._scrollbarWidth+"px"),this._isBodyOverflowing&&!t&&(this._element.style.paddingRight=this._scrollbarWidth+"px")},a.prototype._resetAdjustments=function(){this._element.style.paddingLeft="",this._element.style.paddingRight=""},a.prototype._checkScrollbar=function(){this._isBodyOverflowing=document.body.clientWidth=n){var i=this._targets[this._targets.length-1];this._activeTarget!==i&&this._activate(i)}else{if(this._activeTarget&&t0)return this._activeTarget=null,void this._clear();for(var o=this._offsets.length;o--;)this._activeTarget!==this._targets[o]&&t>=this._offsets[o]&&(void 0===this._offsets[o+1]||t .dropdown-menu .active"},l=function(){function e(t){n(this,e),this._element=t}return e.prototype.show=function(){var e=this;if(!(this._element.parentNode&&this._element.parentNode.nodeType===Node.ELEMENT_NODE&&t(this._element).hasClass(s.ACTIVE)||t(this._element).hasClass(s.DISABLED))){var n=void 0,o=void 0,l=t(this._element).closest(a.NAV_LIST_GROUP)[0],h=r.getSelectorFromElement(this._element);l&&(o=t.makeArray(t(l).find(a.ACTIVE)),o=o[o.length-1]);var c=t.Event(i.HIDE,{relatedTarget:this._element}),u=t.Event(i.SHOW,{relatedTarget:o});if(o&&t(o).trigger(c),t(this._element).trigger(u),!u.isDefaultPrevented()&&!c.isDefaultPrevented()){h&&(n=t(h)[0]),this._activate(this._element,l);var d=function(){var n=t.Event(i.HIDDEN,{relatedTarget:e._element}),r=t.Event(i.SHOWN,{relatedTarget:o});t(o).trigger(n),t(e._element).trigger(r)};n?this._activate(n,n.parentNode,d):d()}}},e.prototype.dispose=function(){t.removeData(this._element,"bs.tab"),this._element=null},e.prototype._activate=function(e,n,i){var o=this,l=t(n).find(a.ACTIVE)[0],h=i&&r.supportsTransitionEnd()&&l&&t(l).hasClass(s.FADE),c=function(){return o._transitionComplete(e,l,h,i)};l&&h?t(l).one(r.TRANSITION_END,c).emulateTransitionEnd(150):c(),l&&t(l).removeClass(s.SHOW)},e.prototype._transitionComplete=function(e,n,i,o){if(n){t(n).removeClass(s.ACTIVE);var l=t(n.parentNode).find(a.DROPDOWN_ACTIVE_CHILD)[0];l&&t(l).removeClass(s.ACTIVE),n.setAttribute("aria-expanded",!1)}if(t(e).addClass(s.ACTIVE),e.setAttribute("aria-expanded",!0),i?(r.reflow(e),t(e).addClass(s.SHOW)):t(e).removeClass(s.FADE),e.parentNode&&t(e.parentNode).hasClass(s.DROPDOWN_MENU)){var h=t(e).closest(a.DROPDOWN)[0];h&&t(h).find(a.DROPDOWN_TOGGLE).addClass(s.ACTIVE),e.setAttribute("aria-expanded",!0)}o&&o()},e._jQueryInterface=function(n){return this.each(function(){var i=t(this),o=i.data("bs.tab");if(o||(o=new e(this),i.data("bs.tab",o)),"string"==typeof n){if(void 0===o[n])throw new Error('No method named "'+n+'"');o[n]()}})},o(e,null,[{key:"VERSION",get:function(){return"4.0.0-alpha.6"}}]),e}();t(document).on(i.CLICK_DATA_API,a.DATA_TOGGLE,function(e){e.preventDefault(),l._jQueryInterface.call(t(this),"show")}),t.fn.tab=l._jQueryInterface,t.fn.tab.Constructor=l,t.fn.tab.noConflict=function(){return t.fn.tab=e,l._jQueryInterface}}(jQuery),function(t){if("undefined"==typeof Popper)throw new Error("Bootstrap tooltips require Popper.js (https://popper.js.org)");var e="tooltip",s=".bs.tooltip",a=t.fn[e],l=new RegExp("(^|\\s)bs-tooltip\\S+","g"),h={animation:"boolean",template:"string",title:"(string|element|function)",trigger:"string",delay:"(number|object)",html:"boolean",selector:"(string|boolean)",placement:"(string|function)",offset:"(number|string)",container:"(string|element|boolean)",fallbackPlacement:"(string|array)"},c={AUTO:"auto",TOP:"top",RIGHT:"right",BOTTOM:"bottom",LEFT:"left"},u={animation:!0,template:'',trigger:"hover focus",title:"",delay:0,html:!1,selector:!1,placement:"top",offset:0,container:!1,fallbackPlacement:"flip"},d={SHOW:"show",OUT:"out"},f={HIDE:"hide"+s,HIDDEN:"hidden"+s,SHOW:"show"+s,SHOWN:"shown"+s,INSERTED:"inserted"+s,CLICK:"click"+s,FOCUSIN:"focusin"+s,FOCUSOUT:"focusout"+s,MOUSEENTER:"mouseenter"+s,MOUSELEAVE:"mouseleave"+s},p={FADE:"fade",SHOW:"show"},_={TOOLTIP:".tooltip",TOOLTIP_INNER:".tooltip-inner",ARROW:".arrow"},g={HOVER:"hover",FOCUS:"focus",CLICK:"click",MANUAL:"manual"},m=function(){function a(t,e){n(this,a),this._isEnabled=!0,this._timeout=0,this._hoverState="",this._activeTrigger={},this._popper=null,this.element=t,this.config=this._getConfig(e),this.tip=null,this._setListeners()}return a.prototype.enable=function(){this._isEnabled=!0},a.prototype.disable=function(){this._isEnabled=!1},a.prototype.toggleEnabled=function(){this._isEnabled=!this._isEnabled},a.prototype.toggle=function(e){if(e){var n=this.constructor.DATA_KEY,i=t(e.currentTarget).data(n);i||(i=new this.constructor(e.currentTarget,this._getDelegateConfig()),t(e.currentTarget).data(n,i)),i._activeTrigger.click=!i._activeTrigger.click,i._isWithActiveTrigger()?i._enter(null,i):i._leave(null,i)}else{if(t(this.getTipElement()).hasClass(p.SHOW))return void this._leave(null,this);this._enter(null,this)}},a.prototype.dispose=function(){clearTimeout(this._timeout),t.removeData(this.element,this.constructor.DATA_KEY),t(this.element).off(this.constructor.EVENT_KEY),t(this.element).closest(".modal").off("hide.bs.modal"),this.tip&&t(this.tip).remove(),this._isEnabled=null,this._timeout=null,this._hoverState=null,this._activeTrigger=null,null!==this._popper&&this._popper.destroy(),this._popper=null,this.element=null,this.config=null,this.tip=null},a.prototype.show=function(){var e=this;if("none"===t(this.element).css("display"))throw new Error("Please use show on visible elements");var n=t.Event(this.constructor.Event.SHOW);if(this.isWithContent()&&this._isEnabled){t(this.element).trigger(n);var i=t.contains(this.element.ownerDocument.documentElement,this.element);if(n.isDefaultPrevented()||!i)return;var o=this.getTipElement(),s=r.getUID(this.constructor.NAME);o.setAttribute("id",s),this.element.setAttribute("aria-describedby",s),this.setContent(),this.config.animation&&t(o).addClass(p.FADE);var l="function"==typeof this.config.placement?this.config.placement.call(this,o,this.element):this.config.placement,h=this._getAttachment(l);this.addAttachmentClass(h);var c=!1===this.config.container?document.body:t(this.config.container);t(o).data(this.constructor.DATA_KEY,this),t.contains(this.element.ownerDocument.documentElement,this.tip)||t(o).appendTo(c),t(this.element).trigger(this.constructor.Event.INSERTED),this._popper=new Popper(this.element,o,{placement:h,modifiers:{offset:{offset:this.config.offset},flip:{behavior:this.config.fallbackPlacement},arrow:{element:_.ARROW}},onCreate:function(t){t.originalPlacement!==t.placement&&e._handlePopperPlacementChange(t)},onUpdate:function(t){e._handlePopperPlacementChange(t)}}),t(o).addClass(p.SHOW),"ontouchstart"in document.documentElement&&t("body").children().on("mouseover",null,t.noop);var u=function(){e.config.animation&&e._fixTransition();var n=e._hoverState;e._hoverState=null,t(e.element).trigger(e.constructor.Event.SHOWN),n===d.OUT&&e._leave(null,e)};r.supportsTransitionEnd()&&t(this.tip).hasClass(p.FADE)?t(this.tip).one(r.TRANSITION_END,u).emulateTransitionEnd(a._TRANSITION_DURATION):u()}},a.prototype.hide=function(e){var n=this,i=this.getTipElement(),o=t.Event(this.constructor.Event.HIDE),s=function(){n._hoverState!==d.SHOW&&i.parentNode&&i.parentNode.removeChild(i),n._cleanTipClass(),n.element.removeAttribute("aria-describedby"),t(n.element).trigger(n.constructor.Event.HIDDEN),null!==n._popper&&n._popper.destroy(),e&&e()};t(this.element).trigger(o),o.isDefaultPrevented()||(t(i).removeClass(p.SHOW),"ontouchstart"in document.documentElement&&t("body").children().off("mouseover",null,t.noop),this._activeTrigger[g.CLICK]=!1,this._activeTrigger[g.FOCUS]=!1,this._activeTrigger[g.HOVER]=!1,r.supportsTransitionEnd()&&t(this.tip).hasClass(p.FADE)?t(i).one(r.TRANSITION_END,s).emulateTransitionEnd(150):s(),this._hoverState="")},a.prototype.update=function(){null!==this._popper&&this._popper.scheduleUpdate()},a.prototype.isWithContent=function(){return Boolean(this.getTitle())},a.prototype.addAttachmentClass=function(e){t(this.getTipElement()).addClass("bs-tooltip-"+e)},a.prototype.getTipElement=function(){return this.tip=this.tip||t(this.config.template)[0]},a.prototype.setContent=function(){var e=t(this.getTipElement());this.setElementContent(e.find(_.TOOLTIP_INNER),this.getTitle()),e.removeClass(p.FADE+" "+p.SHOW)},a.prototype.setElementContent=function(e,n){var o=this.config.html;"object"===(void 0===n?"undefined":i(n))&&(n.nodeType||n.jquery)?o?t(n).parent().is(e)||e.empty().append(n):e.text(t(n).text()):e[o?"html":"text"](n)},a.prototype.getTitle=function(){var t=this.element.getAttribute("data-original-title");return t||(t="function"==typeof this.config.title?this.config.title.call(this.element):this.config.title),t},a.prototype._getAttachment=function(t){return c[t.toUpperCase()]},a.prototype._setListeners=function(){var e=this;this.config.trigger.split(" ").forEach(function(n){if("click"===n)t(e.element).on(e.constructor.Event.CLICK,e.config.selector,function(t){return e.toggle(t)});else if(n!==g.MANUAL){var i=n===g.HOVER?e.constructor.Event.MOUSEENTER:e.constructor.Event.FOCUSIN,o=n===g.HOVER?e.constructor.Event.MOUSELEAVE:e.constructor.Event.FOCUSOUT;t(e.element).on(i,e.config.selector,function(t){return e._enter(t)}).on(o,e.config.selector,function(t){return e._leave(t)})}t(e.element).closest(".modal").on("hide.bs.modal",function(){return e.hide()})}),this.config.selector?this.config=t.extend({},this.config,{trigger:"manual",selector:""}):this._fixTitle()},a.prototype._fixTitle=function(){var t=i(this.element.getAttribute("data-original-title"));(this.element.getAttribute("title")||"string"!==t)&&(this.element.setAttribute("data-original-title",this.element.getAttribute("title")||""),this.element.setAttribute("title",""))},a.prototype._enter=function(e,n){var i=this.constructor.DATA_KEY;(n=n||t(e.currentTarget).data(i))||(n=new this.constructor(e.currentTarget,this._getDelegateConfig()),t(e.currentTarget).data(i,n)),e&&(n._activeTrigger["focusin"===e.type?g.FOCUS:g.HOVER]=!0),t(n.getTipElement()).hasClass(p.SHOW)||n._hoverState===d.SHOW?n._hoverState=d.SHOW:(clearTimeout(n._timeout),n._hoverState=d.SHOW,n.config.delay&&n.config.delay.show?n._timeout=setTimeout(function(){n._hoverState===d.SHOW&&n.show()},n.config.delay.show):n.show())},a.prototype._leave=function(e,n){var i=this.constructor.DATA_KEY;(n=n||t(e.currentTarget).data(i))||(n=new this.constructor(e.currentTarget,this._getDelegateConfig()),t(e.currentTarget).data(i,n)),e&&(n._activeTrigger["focusout"===e.type?g.FOCUS:g.HOVER]=!1),n._isWithActiveTrigger()||(clearTimeout(n._timeout),n._hoverState=d.OUT,n.config.delay&&n.config.delay.hide?n._timeout=setTimeout(function(){n._hoverState===d.OUT&&n.hide()},n.config.delay.hide):n.hide())},a.prototype._isWithActiveTrigger=function(){for(var t in this._activeTrigger)if(this._activeTrigger[t])return!0;return!1},a.prototype._getConfig=function(n){return(n=t.extend({},this.constructor.Default,t(this.element).data(),n)).delay&&"number"==typeof n.delay&&(n.delay={show:n.delay,hide:n.delay}),n.title&&"number"==typeof n.title&&(n.title=n.title.toString()),n.content&&"number"==typeof n.content&&(n.content=n.content.toString()),r.typeCheckConfig(e,n,this.constructor.DefaultType),n},a.prototype._getDelegateConfig=function(){var t={};if(this.config)for(var e in this.config)this.constructor.Default[e]!==this.config[e]&&(t[e]=this.config[e]);return t},a.prototype._cleanTipClass=function(){var e=t(this.getTipElement()),n=e.attr("class").match(l);null!==n&&n.length>0&&e.removeClass(n.join(""))},a.prototype._handlePopperPlacementChange=function(t){this._cleanTipClass(),this.addAttachmentClass(this._getAttachment(t.placement))},a.prototype._fixTransition=function(){var e=this.getTipElement(),n=this.config.animation;null===e.getAttribute("x-placement")&&(t(e).removeClass(p.FADE),this.config.animation=!1,this.hide(),this.show(),this.config.animation=n)},a._jQueryInterface=function(e){return this.each(function(){var n=t(this).data("bs.tooltip"),o="object"===(void 0===e?"undefined":i(e))&&e;if((n||!/dispose|hide/.test(e))&&(n||(n=new a(this,o),t(this).data("bs.tooltip",n)),"string"==typeof e)){if(void 0===n[e])throw new Error('No method named "'+e+'"');n[e]()}})},o(a,null,[{key:"VERSION",get:function(){return"4.0.0-alpha.6"}},{key:"Default",get:function(){return u}},{key:"NAME",get:function(){return e}},{key:"DATA_KEY",get:function(){return"bs.tooltip"}},{key:"Event",get:function(){return f}},{key:"EVENT_KEY",get:function(){return s}},{key:"DefaultType",get:function(){return h}}]),a}();return t.fn[e]=m._jQueryInterface,t.fn[e].Constructor=m,t.fn[e].noConflict=function(){return t.fn[e]=a,m._jQueryInterface},m}(jQuery));!function(r){var a="popover",l=".bs.popover",h=r.fn[a],c=new RegExp("(^|\\s)bs-popover\\S+","g"),u=r.extend({},s.Default,{placement:"right",trigger:"click",content:"",template:''}),d=r.extend({},s.DefaultType,{content:"(string|element|function)"}),f={FADE:"fade",SHOW:"show"},p={TITLE:".popover-header",CONTENT:".popover-body"},_={HIDE:"hide"+l,HIDDEN:"hidden"+l,SHOW:"show"+l,SHOWN:"shown"+l,INSERTED:"inserted"+l,CLICK:"click"+l,FOCUSIN:"focusin"+l,FOCUSOUT:"focusout"+l,MOUSEENTER:"mouseenter"+l,MOUSELEAVE:"mouseleave"+l},g=function(s){function h(){return n(this,h),t(this,s.apply(this,arguments))}return e(h,s),h.prototype.isWithContent=function(){return this.getTitle()||this._getContent()},h.prototype.addAttachmentClass=function(t){r(this.getTipElement()).addClass("bs-popover-"+t)},h.prototype.getTipElement=function(){return this.tip=this.tip||r(this.config.template)[0]},h.prototype.setContent=function(){var t=r(this.getTipElement());this.setElementContent(t.find(p.TITLE),this.getTitle()),this.setElementContent(t.find(p.CONTENT),this._getContent()),t.removeClass(f.FADE+" "+f.SHOW)},h.prototype._getContent=function(){return this.element.getAttribute("data-content")||("function"==typeof this.config.content?this.config.content.call(this.element):this.config.content)},h.prototype._cleanTipClass=function(){var t=r(this.getTipElement()),e=t.attr("class").match(c);null!==e&&e.length>0&&t.removeClass(e.join(""))},h._jQueryInterface=function(t){return this.each(function(){var e=r(this).data("bs.popover"),n="object"===(void 0===t?"undefined":i(t))?t:null;if((e||!/destroy|hide/.test(t))&&(e||(e=new h(this,n),r(this).data("bs.popover",e)),"string"==typeof t)){if(void 0===e[t])throw new Error('No method named "'+t+'"');e[t]()}})},o(h,null,[{key:"VERSION",get:function(){return"4.0.0-alpha.6"}},{key:"Default",get:function(){return u}},{key:"NAME",get:function(){return a}},{key:"DATA_KEY",get:function(){return"bs.popover"}},{key:"Event",get:function(){return _}},{key:"EVENT_KEY",get:function(){return l}},{key:"DefaultType",get:function(){return d}}]),h}(s);r.fn[a]=g._jQueryInterface,r.fn[a].Constructor=g,r.fn[a].noConflict=function(){return r.fn[a]=h,g._jQueryInterface}}(jQuery)}(); diff --git a/website/build_site.sh b/website/build_site.sh index 08fb34fbb3..dfae6f62da 100755 --- a/website/build_site.sh +++ b/website/build_site.sh @@ -62,4 +62,3 @@ fi #git add . #git commit -m "Automatic Site Publish by Buildbot" #git push origin asf-site - diff --git a/website/community/coc.md b/website/community/coc.md index 03b0607878..8f1a30efd6 100644 --- a/website/community/coc.md +++ b/website/community/coc.md @@ -49,4 +49,3 @@ This Code of Conduct applies within all community spaces, including online forum ## Attribution This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 2.0, available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. - diff --git a/website/community/mailing-lists.md b/website/community/mailing-lists.md index 10d3ed0df9..e5046b1d7e 100644 --- a/website/community/mailing-lists.md +++ b/website/community/mailing-lists.md @@ -2,7 +2,7 @@ layout: page title: Mailing Lists, IRC and Archives - + --- # General @@ -16,9 +16,9 @@ list so we are able to monitor what happens on the wiki and in svn. ## How to subscribe/unsubscribe -Follow the links below, or send mail manually, with empty subject and body. +Follow the links below, or send mail manually, with empty subject and body. -The pattern for subscribing and unsubscribing to mailing lists at the ASF +The pattern for subscribing and unsubscribing to mailing lists at the ASF is <list-name>-<action>@<project.>apache.org. @@ -27,7 +27,7 @@ is <list-name>-<action>@<project.>apache.org. This list is for users of Mahout to ask questions, share knowledge, and discuss issues. Do send mail to this list with usage and configuration questions and problems. Also, please send questions to this list to verify -your problem before filing issues in JIRA. +your problem before filing issues in JIRA. * [Subscribe](mailto:user-subscribe@mahout.apache.org) * [Unsubscribe](mailto:user-unsubscribe@mahout.apache.org) @@ -38,16 +38,16 @@ your problem before filing issues in JIRA. This is the list where participating developers of the Mahout project meet and discuss issues concerning Mahout internals, code changes/additions, etc. Do not send mail to this list with usage questions or configuration -questions and problems. +questions and problems. -Discussion list: +Discussion list: * [Subscribe](mailto:dev-subscribe@mahout.apache.org) -- Do not send mail to this list with usage questions or configuration -questions and problems. +questions and problems. * [Unsubscribe](mailto:dev-unsubscribe@mahout.apache.org) -Commit notifications: +Commit notifications: * [Subscribe](mailto:commits-subscribe@mahout.apache.org) * [Unsubscribe](mailto:commits-unsubscribe@mahout.apache.org) diff --git a/website/community/who-we-are.md b/website/community/who-we-are.md index ade33291fd..9c2bf89952 100644 --- a/website/community/who-we-are.md +++ b/website/community/who-we-are.md @@ -2,7 +2,7 @@ layout: page title: Who Are We? - + --- @@ -15,36 +15,36 @@ Apache Mahout is maintained by a team of volunteer developers. (Please keep the list below in alphabetical order by first name.) -Name | Mail | PMC | Comment +Name | Mail | PMC | Comment ------------------------|------------------:|------:|---:| Anand Avati | avati@... | No | @anandavati Andrew Musselman | akm@... | Yes | @akm -Andrew Palumbo | apalumbo@... | Yes | -Dan Filimon | dfilimon@... | No | +Andrew Palumbo | apalumbo@... | Yes | +Dan Filimon | dfilimon@... | No | Dmitriy Lyubimov | dlyubimov@... | No | (Emeritus PMC) Drew Farris | drew@... | No | (Emeritus PMC) -Dustin VanStee | vanstee@... | No | +Dustin VanStee | vanstee@... | No | Ellen Friedman | ellenf@... | No | @Ellen_Friedman -Frank Scholten | frankscholten@... | No | -Gokhan Capan | gcapan@... | No | +Frank Scholten | frankscholten@... | No | +Gokhan Capan | gcapan@... | No | Grant Ingersoll | gsingers@... | No | (Emeritus PMC) @gsingers -Holden Karau | holden@... | No | +Holden Karau | holden@... | No | Isabel Drost-Fromm | isabel@... | Yes | -Jacob Alexander Mannix | jmannix@... | Yes | +Jacob Alexander Mannix | jmannix@... | Yes | Jeff Eastman | jeastman@... | No | (Emeritus PMC) Krishna Dave | krishnadave829@...| No | @krishnadave -Nikolay Sakharnykh | nsakharnykh@... | No | +Nikolay Sakharnykh | nsakharnykh@... | No | Paritosh Ranjan | pranjan@... | Yes | @paritoshranjan -Pat Ferrel | pat@... | Yes | @occam -Robin Anil | robinanil@... | Yes | +Pat Ferrel | pat@... | Yes | @occam +Robin Anil | robinanil@... | Yes | Sean Owen | srowen@... | No | (Emeritus PMC) -Sebastian Schelter | ssc@... | Yes | -Shannon Quinn | squinn@... | Yes | PMC Chair @magsol@quinnwitz.house +Sebastian Schelter | ssc@... | Yes | +Shannon Quinn | squinn@... | Yes | PMC Chair @magsol@quinnwitz.house Stevo Slavić | sslavic@... | Yes | @sslavic Suneel Marthi | smarthi@... | No | (Emeritus PMC) @suneelmarthi -Ted Dunning | tdunning@... | Yes | -Tom Pierce | tcp@... | No | -Trevor Grant | rawkintrevo@... | Yes | @rawkintrevo +Ted Dunning | tdunning@... | Yes | +Tom Pierce | tcp@... | No | +Trevor Grant | rawkintrevo@... | Yes | @rawkintrevo ## Emeritus Committers diff --git a/website/docs/README.md b/website/docs/README.md index 3ac42a1026..487b7c99a5 100755 --- a/website/docs/README.md +++ b/website/docs/README.md @@ -29,4 +29,3 @@ Tutorials, Algorithms, MR-Tutorials, and MR-Algorithsm each have an accordion si ![landing](screenshots/mr-algos.png) - diff --git a/website/docs/latest/algorithms/regression/serial-correlation/dw-test.md b/website/docs/latest/algorithms/regression/serial-correlation/dw-test.md index 1e5d007989..94d4f7fede 100644 --- a/website/docs/latest/algorithms/regression/serial-correlation/dw-test.md +++ b/website/docs/latest/algorithms/regression/serial-correlation/dw-test.md @@ -19,23 +19,22 @@ Implementation is based off of the `durbinWatsonTest` function in the [`car`](ht ### Example #### R Prototype - + library(car) residuals <- seq(0, 4.9, 0.1) ## perform Durbin-Watson test durbinWatsonTest(residuals) - + #### In Apache Mahout - + // A DurbinWatson Test must be performed on a model. The model does not matter. val drmX = drmParallelize( dense((0 until 50).toArray.map( t => Math.pow(-1.0, t)) ) ).t val drmY = drmX + err1 + 1 var model = new OrdinaryLeastSquares[Int]().fit(drmX, drmY) - // end arbitrary model - + // end arbitrary model + val err1 = drmParallelize( dense((0.0 until 5.0 by 0.1).toArray) ).t val syntheticResiduals = err1 model = AutocorrelationTests.DurbinWatson(model, syntheticResiduals) val myAnswer: Double = model.testResults.getOrElse('durbinWatsonTestStatistic, -1.0).asInstanceOf[Double] - diff --git a/website/docs/latest/changelog.md b/website/docs/latest/changelog.md index 5e4ca48178..0bbcd746c9 100755 --- a/website/docs/latest/changelog.md +++ b/website/docs/latest/changelog.md @@ -5,7 +5,7 @@ title: Changelog ## Changelog -Public releases are all root nodes. +Public releases are all root nodes. Incremental version bumps that were not released publicly are nested where appropriate. P.S. If there is a standard (popular) changelog format, please let me know. @@ -21,7 +21,7 @@ P.S. If there is a standard (popular) changelog format, please let me know. - **Bug Fixes** - @xuhdev fixes install bug which does not overwrite theme even if saying 'yes'. -- **0.2.13 : 2012.03.24** +- **0.2.13 : 2012.03.24** - **Features** - 0.2.13 : @mjpieters Updates pages_list helper to only show pages having a title. - 0.2.12 : @sway recommends showing page tagline only if tagline is set. @@ -30,11 +30,11 @@ P.S. If there is a standard (popular) changelog format, please let me know. - **Bug Fixes** - 0.2.10 : @koriroys fixes typo in atom feed -- **0.2.9 : 2012.03.01** +- **0.2.9 : 2012.03.01** - **Bug Fixes** - 0.2.9 : @alishutc Fixes the error on post creation if date was not specified. -- **0.2.8 : 2012.03.01** +- **0.2.8 : 2012.03.01** - **Features** - 0.2.8 : @metalelf0 Added option to specify a custom date when creating post. - 0.2.7 : @daz Updates twitter theme framework to use 2.x while still maintaining core layout. #50 @@ -48,7 +48,7 @@ P.S. If there is a standard (popular) changelog format, please let me know. - 0.2.2 : @sstar Resolved an issue preventing disabling comments for individual pages #44 - 0.2.1 : Resolve incorrect HOME\_PATH/BASE\_PATH settings -- **0.2.0 : 2012.02.01** +- **0.2.0 : 2012.02.01** Features - Add Theme Packages v 0.1.0 All themes should be tracked and maintained outside of JB core. @@ -59,17 +59,16 @@ P.S. If there is a standard (popular) changelog format, please let me know. - 0.1.2 : @jamesFleeting adds facebook comments support - 0.1.1 : @SegFaultAX adds tagline as site-wide configuration -- **0.1.0 : 2012.01.24** - First major versioned release. - Features +- **0.1.0 : 2012.01.24** + First major versioned release. + Features - Standardize Public API - Use name-spacing and modulation where possible. - Ability to override public methods with custom code. - Publish the theme API. - Ship with comments, analytics integration. - -- **0.0.1 : 2011.12.30** + +- **0.0.1 : 2011.12.30** First public release, lots of updates =p Thank you everybody for dealing with the fast changes and helping me work out the API to a manageable state. - diff --git a/website/documentation/tutorials/cco-lastfm/cco-lastfm.scala b/website/documentation/tutorials/cco-lastfm/cco-lastfm.scala index 709ab2a785..9264f03b4d 100644 --- a/website/documentation/tutorials/cco-lastfm/cco-lastfm.scala +++ b/website/documentation/tutorials/cco-lastfm/cco-lastfm.scala @@ -22,7 +22,7 @@ import org.apache.mahout.sparkbindings.indexeddataset.IndexedDatasetSpark -// We need to turn our raw text files into RDD[(String, String)] +// We need to turn our raw text files into RDD[(String, String)] val userTagsRDD = sc.textFile("/path/to/lastfm/user_taggedartists.dat").map(line => line.split("\t")).map(a => (a(0), a(2))).filter(_._1 != "userID") val userTagsIDS = IndexedDatasetSpark.apply(userTagsRDD)(sc) @@ -109,4 +109,4 @@ println(kilroysRecs(::, 0).toMap.toList.sortWith(_._2 > _._2).take(5)) * is actually a pretty dope idea. * * Solving for which bands a user might like is left as an exercise to the reader. - */ \ No newline at end of file + */ diff --git a/website/documentation/tutorials/misc/contributing-algos/Foo.scala b/website/documentation/tutorials/misc/contributing-algos/Foo.scala index 171993f7f5..91164f7257 100644 --- a/website/documentation/tutorials/misc/contributing-algos/Foo.scala +++ b/website/documentation/tutorials/misc/contributing-algos/Foo.scala @@ -72,4 +72,4 @@ class FooModel[K] extends RegressorModel[K] { } } } -} \ No newline at end of file +} diff --git a/website/documentation/users/dim-reduction/ssvd.page/ssvd.R b/website/documentation/users/dim-reduction/ssvd.page/ssvd.R index fa5fa846da..5f9cf3123f 100644 --- a/website/documentation/users/dim-reduction/ssvd.page/ssvd.R +++ b/website/documentation/users/dim-reduction/ssvd.page/ssvd.R @@ -1,6 +1,6 @@ # standard SSVD -ssvd.svd <- function(x, k, p=25, qiter=0 ) { +ssvd.svd <- function(x, k, p=25, qiter=0 ) { a <- as.matrix(x) m <- nrow(a) @@ -17,7 +17,7 @@ q <- qr.Q(qr(y)) b<- t(q) %*% a #power iterations -for ( i in 1:qiter ) { +for ( i in 1:qiter ) { y <- a %*% t(b) q <- qr.Q(qr(y)) b <- t(q) %*% a @@ -40,7 +40,7 @@ return(res) #SSVD with Q=YR^-1 substitute. # this is just a simulation, because it is suboptimal to verify the actual result -ssvd.svd1 <- function(x, k, p=25, qiter=0 ) { +ssvd.svd1 <- function(x, k, p=25, qiter=0 ) { a <- as.matrix(x) m <- nrow(a) @@ -58,10 +58,10 @@ yty <- t(y) %*% y R <- chol(yty, pivot = T) q <- y %*% solve(R) -b<- t( q ) %*% a +b<- t( q ) %*% a #power iterations -for ( i in 1:qiter ) { +for ( i in 1:qiter ) { y <- a %*% t(b) yty <- t(y) %*% y @@ -88,7 +88,7 @@ return(res) ############# ## ssvd with pci options -ssvd.cpca <- function ( x, k, p=25, qiter=0, fixY=T ) { +ssvd.cpca <- function ( x, k, p=25, qiter=0, fixY=T ) { a <- as.matrix(x) m <- nrow(a) @@ -105,7 +105,7 @@ omega <- matrix ( rnorm(r*n), nrow=n, ncol=r) y <- a %*% omega #fix y -if ( fixY ) { +if ( fixY ) { #debug cat ("fixing Y...\n"); @@ -118,27 +118,27 @@ q <- qr.Q(qr(y)) b<- t(q) %*% a -# compute sum of q rows +# compute sum of q rows s_q <- cbind(colSums(q)) # compute B*xi -# of course in MR implementation +# of course in MR implementation # it will be collected as sums of ( B[,i] * xi[i] ) and reduced after. s_b <- b %*% cbind(xi) #power iterations -for ( i in 1:qiter ) { +for ( i in 1:qiter ) { - # fix b - b <- b - s_q %*% rbind(xi) + # fix b + b <- b - s_q %*% rbind(xi) y <- a %*% t(b) - # fix y - if ( fixY ) + # fix y + if ( fixY ) for (i in 1:r ) y[,i]<- y[,i]-s_b[i] - + q <- qr.Q(qr(y)) b <- t(q) %*% a @@ -173,9 +173,3 @@ res$v <- (t(b- s_q %*% rbind(xi) ) %*% e$vectors %*% diag(1/e$values))[,1:k] return(res) } - - - - - - diff --git a/website/documentation/users/index.md b/website/documentation/users/index.md index c4b156ac13..f112faa5e9 100644 --- a/website/documentation/users/index.md +++ b/website/documentation/users/index.md @@ -5,12 +5,12 @@ title: User's Guide # Apache Mahout User's Guide -Apache Mahout is a powerful, scalable, and versatile machine learning library designed for distributed data processing. +Apache Mahout is a powerful, scalable, and versatile machine learning library designed for distributed data processing. It offers a comprehensive set of algorithms for various tasks, including Classification, clustering, recommendation, and -pattern mining. Built on top of the Apache Hadoop ecosystem, Mahout leverages MapReduce and Spark to enable data +pattern mining. Built on top of the Apache Hadoop ecosystem, Mahout leverages MapReduce and Spark to enable data processing on large-scale datasets. -In this User's Guide, we provide an overview of Apache Mahout, its key features, and how to get started with using the +In this User's Guide, we provide an overview of Apache Mahout, its key features, and how to get started with using the library for your machine learning projects. ## Key Features @@ -28,7 +28,7 @@ library for your machine learning projects. 4. **Model Training and Evaluation**: Understand how to train, validate, and evaluate machine learning models using Mahout's tools and best practices. 5. **Deployment**: Explore various options for deploying your trained models, such as integrating with web services or embedding within your applications. -By following this User's Guide, you will gain the necessary knowledge and skills to effectively leverage Apache Mahout +By following this User's Guide, you will gain the necessary knowledge and skills to effectively leverage Apache Mahout for your machine learning projects, harnessing the power of big data processing to achieve better results. ## Index diff --git a/website/download/downloads.md b/website/download/downloads.md index dc9a1e1863..336e1c6a06 100644 --- a/website/download/downloads.md +++ b/website/download/downloads.md @@ -8,7 +8,7 @@ title: Downloads # Official Release Apache Mahout is an official Apache project and thus available from any of -the Apache mirrors. The latest Mahout release is available for download at: +the Apache mirrors. The latest Mahout release is available for download at: * [Download Latest](http://www.apache.org/dist/mahout) * [Release Archive](http://archive.apache.org/dist/mahout) @@ -42,4 +42,3 @@ Official releases are usually created when the developers feel there are sufficient changes, improvements and bug fixes to warrant a release. Watch the Mailing lists for latest release discussions and check the Github repo. - diff --git a/website/download/quickstart.md b/website/download/quickstart.md index 428ad692c3..b7ed6e08ab 100644 --- a/website/download/quickstart.md +++ b/website/download/quickstart.md @@ -18,7 +18,7 @@ layout: page title: Quickstart - + --- # Mahout MapReduce Overview @@ -47,7 +47,7 @@ If you would like to import the latest release of mahout into a java project, ad For a full list of Mahout's features see our [Features by Engine](http://mahout.apache.org/users/basics/algorithms.html) page. - + ## Using Mahout Mahout has prepared a bunch of examples and tutorials for users to quickly learn how to use its machine learning algorithms. @@ -66,10 +66,10 @@ Check the [Synthetic data](/users/clustering/clustering-of-synthetic-control-dat If you are interested in how to train a **Naive Bayes** model, look at the [20 newsgroups](/users/classification/twenty-newsgroups.html) example. -If you plan to build a **Hidden Markov Model** for speech recognition, the example [here](/users/classification/hidden-markov-models.html) might be instructive. +If you plan to build a **Hidden Markov Model** for speech recognition, the example [here](/users/classification/hidden-markov-models.html) might be instructive. Or you could build a **Random Forest** model by following this [quick start page](/users/classification/partial-implementation.html). -#### Working with Text +#### Working with Text -If you need to convert raw text into word vectors as input to clustering or classification algorithms, please refer to this page on [how to create vectors from text](/users/basics/creating-vectors-from-text.html). \ No newline at end of file +If you need to convert raw text into word vectors as input to clustering or classification algorithms, please refer to this page on [how to create vectors from text](/users/basics/creating-vectors-from-text.html). diff --git a/website/index.md b/website/index.md index f9a51d8359..185dd9a9a1 100644 --- a/website/index.md +++ b/website/index.md @@ -13,4 +13,3 @@ Apache Mahout(TM) is a distributed linear algebra framework and {% for post in site.posts limit: 5 %} {% include posts/post-preview.html %} {% endfor %} - diff --git a/website/quantum-computing-primer/01_introduction/index.md b/website/quantum-computing-primer/01_introduction/index.md index dc8921e195..2b66b97927 100644 --- a/website/quantum-computing-primer/01_introduction/index.md +++ b/website/quantum-computing-primer/01_introduction/index.md @@ -17,8 +17,8 @@ Quantum computing is a revolutionary approach to computation that leverages the ### Why Quantum Computing Matters: Quantum computing has the potential to revolutionize fields such as cryptography, optimization, and material science. For example, quantum algorithms like **Shor's algorithm** can factorize large numbers exponentially faster than classical algorithms, posing a threat to current cryptographic systems. Similarly, **Grover's algorithm** can search unsorted databases quadratically faster than classical methods. - ---- + +--- ## 1.2 Why Quantum Computing? @@ -39,38 +39,38 @@ Quantum computing is not just a theoretical concept; it has practical implicatio ### 5. **Machine Learning**: - Quantum machine learning algorithms promise to accelerate training and improve model performance for specific tasks. ---- +--- ## 1.3 Quantum Computing vs. Classical Computing -| Feature | Classical Computing | Quantum Computing | -|------------------------|-----------------------------------|----------------------------------| -| **Basic Unit** | Bit (0 or 1) | Qubit (superposition of 0 and 1)| -| **State Representation**| Deterministic | Probabilistic | -| **Operations** | Logic gates (AND, OR, NOT, etc.) | Quantum gates (X, Y, Z, H, etc.)| -| **Parallelism** | Limited by CPU cores | Exponential parallelism via superposition | -| **Error Correction** | Well-established | Still an active area of research| - ---- +| Feature | Classical Computing | Quantum Computing | +|------------------------|-----------------------------------|----------------------------------| +| **Basic Unit** | Bit (0 or 1) | Qubit (superposition of 0 and 1)| +| **State Representation**| Deterministic | Probabilistic | +| **Operations** | Logic gates (AND, OR, NOT, etc.) | Quantum gates (X, Y, Z, H, etc.)| +| **Parallelism** | Limited by CPU cores | Exponential parallelism via superposition | +| **Error Correction** | Well-established | Still an active area of research| + +--- ## 1.4 Getting Started with Quantum Computing Using `qumat` To begin your journey into quantum computing, you'll use the `qumat` library, which provides a simple and unified interface for working with quantum circuits across different backends (e.g., Amazon Braket, Cirq, Qiskit). Here's a quick example to get you started: -```python -from qumat import QuMat - -# Initialize a quantum circuit with 1 qubit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) -qc.create_empty_circuit(1) - -# Apply a Hadamard gate to create a superposition -qc.apply_hadamard_gate(0) - -# Execute the circuit and measure the result -result = qc.execute_circuit() -print(result) +```python +from qumat import QuMat + +# Initialize a quantum circuit with 1 qubit +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) +qc.create_empty_circuit(1) + +# Apply a Hadamard gate to create a superposition +qc.apply_hadamard_gate(0) + +# Execute the circuit and measure the result +result = qc.execute_circuit() +print(result) ``` In this example, we: @@ -90,4 +90,4 @@ This is just the beginning! In the next sections, you'll dive deeper into quantu * Quantum computing has the potential to solve problems that are intractable for classical computers. * The `qumat` library provides a simple way to explore quantum computing concepts and algorithms. -In the next section, we'll set up your environment and explore the basics of quantum circuits using `qumat`. \ No newline at end of file +In the next section, we'll set up your environment and explore the basics of quantum circuits using `qumat`. diff --git a/website/quantum-computing-primer/03_qubits/index.md b/website/quantum-computing-primer/03_qubits/index.md index a8761f8963..aaf61a9656 100644 --- a/website/quantum-computing-primer/03_qubits/index.md +++ b/website/quantum-computing-primer/03_qubits/index.md @@ -12,8 +12,8 @@ In classical computing, the fundamental unit of information is the **bit**, whic $$|\psi\rangle = \alpha|0\rangle + \beta|1\rangle$$ {% raw %} -where $\alpha$ and $\beta$ are complex numbers representing the probability -amplitudes of the qubit being in the $|0\rangle$ and $|1\rangle$ states, +where $\alpha$ and $\beta$ are complex numbers representing the probability +amplitudes of the qubit being in the $|0\rangle$ and $|1\rangle$ states, respectively. The probabilities of measuring the qubit in either state are given by $|\alpha|^2$ and $|\beta|^2$, and they must satisfy the normalization condition: {% endraw %} @@ -22,14 +22,14 @@ $$|\alpha|^2 + |\beta|^2 = 1$$ ## 3.2 Representing Qubits -Qubits can be visualized using the **Bloch sphere**, a geometric representation -of the quantum state of a single qubit. The Bloch sphere is a unit sphere where -the north and south poles represent the $|0\rangle$ and $|1\rangle$ states, -respectively. Any point on the surface of the sphere represents a valid quantum +Qubits can be visualized using the **Bloch sphere**, a geometric representation +of the quantum state of a single qubit. The Bloch sphere is a unit sphere where +the north and south poles represent the $|0\rangle$ and $|1\rangle$ states, +respectively. Any point on the surface of the sphere represents a valid quantum state of the qubit. -The state of a qubit can also be described using a **state vector** in a -two-dimensional complex vector space. For example, the state $|0\rangle$ is +The state of a qubit can also be described using a **state vector** in a +two-dimensional complex vector space. For example, the state $|0\rangle$ is represented as: $$|0\rangle = \begin{pmatrix} 1 \\ 0 \end{pmatrix}$$ @@ -44,28 +44,28 @@ $$|1\rangle = \begin{pmatrix} 0 \\ 1 \end{pmatrix}$$ In `qumat`, qubits are created by initializing a quantum circuit with a specified number of qubits. The `create_empty_circuit` function is used to create a circuit with a given number of qubits. Here's an example of creating a quantum circuit with a single qubit: -```python -from qumat import QuMat - -# Initialize the quantum circuit with a single qubit -backend_config = { - 'backend_name': 'qiskit', # Choose the backend (e.g., 'qiskit', 'cirq', 'amazon_braket') - 'backend_options': { - 'simulator_type': 'qasm_simulator', # Type of simulator - 'shots': 1000 # Number of shots (measurements) - } -} - -qc = QuMat(backend_config) -qc.create_empty_circuit(1) # Create a circuit with 1 qubit +```python +from qumat import QuMat + +# Initialize the quantum circuit with a single qubit +backend_config = { + 'backend_name': 'qiskit', # Choose the backend (e.g., 'qiskit', 'cirq', 'amazon_braket') + 'backend_options': { + 'simulator_type': 'qasm_simulator', # Type of simulator + 'shots': 1000 # Number of shots (measurements) + } +} + +qc = QuMat(backend_config) +qc.create_empty_circuit(1) # Create a circuit with 1 qubit ``` In this example, we initialize a quantum circuit with one qubit using the qiskit backend. The create_empty_circuit function sets up the circuit, and we can now apply quantum gates to manipulate the qubit. ### Example: Applying a Hadamard Gate -The Hadamard gate ((H)) is a fundamental quantum gate that puts a qubit into a -superposition state. Applying the Hadamard gate to a qubit initially in the +The Hadamard gate ((H)) is a fundamental quantum gate that puts a qubit into a +superposition state. Applying the Hadamard gate to a qubit initially in the $|0\rangle$ state results in the state: $$H|0\rangle = \frac{1}{\sqrt{2}}(|0\rangle + |1\rangle)$$ @@ -73,16 +73,16 @@ $$H|0\rangle = \frac{1}{\sqrt{2}}(|0\rangle + |1\rangle)$$ Here's how you can apply a Hadamard gate to a qubit using qumat: ```python -# Apply the Hadamard gate to the first qubit (index 0) -qc.apply_hadamard_gate(0) - -# Execute the circuit and get the measurement results -result = qc.execute_circuit() -print(result) +# Apply the Hadamard gate to the first qubit (index 0) +qc.apply_hadamard_gate(0) + +# Execute the circuit and get the measurement results +result = qc.execute_circuit() +print(result) ``` -In this example, the Hadamard gate is applied to the qubit at index 0, and the -circuit is executed to obtain the measurement results. The output will show the +In this example, the Hadamard gate is applied to the qubit at index 0, and the +circuit is executed to obtain the measurement results. The output will show the probabilities of measuring the qubit in the $|0\rangle$ and $|1\rangle$ states. ### Visualizing the Circuit diff --git a/website/quantum-computing-primer/04_quantum_gates/index.md b/website/quantum-computing-primer/04_quantum_gates/index.md index 003c0890da..ab5a747b72 100644 --- a/website/quantum-computing-primer/04_quantum_gates/index.md +++ b/website/quantum-computing-primer/04_quantum_gates/index.md @@ -18,30 +18,30 @@ Single-qubit gates operate on a single qubit, changing its state. Some of the mo - **Rotation Gates (Rx, Ry, Rz)**: Rotate the qubit state around the X, Y, or Z axis of the Bloch sphere. ### Example: Applying a Hadamard Gate -```python +```python from qumat import QuMat # Initialize the quantum circuit with 1 qubit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(1) # Apply the Hadamard gate to the first qubit qc.apply_hadamard_gate(0) # Execute the circuit and print the results -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` ## 4.2 Multi-Qubit Gates -Multi-qubit gates operate on two or more qubits, enabling entanglement and more +Multi-qubit gates operate on two or more qubits, enabling entanglement and more complex quantum operations. Some of the most common multi-qubit gates include: -- **CNOT Gate (Controlled-NOT)**: Flips the target qubit if the control qubit is +- **CNOT Gate (Controlled-NOT)**: Flips the target qubit if the control qubit is in the state $|1\rangle$. -- **Toffoli Gate (CCNOT)**: A controlled-controlled-NOT gate that flips the +- **Toffoli Gate (CCNOT)**: A controlled-controlled-NOT gate that flips the target qubit if both control qubits are in the state $|1\rangle$. - **SWAP Gate**: Exchanges the states of two qubits. @@ -57,8 +57,8 @@ qc.apply_hadamard_gate(0) qc.apply_cnot_gate(0, 1) # Execute the circuit and print the results -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` ## 4.3 Applying Gates with `qumat` @@ -80,8 +80,8 @@ qc.apply_ry_gate(0, 3.14159 / 4) qc.apply_rz_gate(0, 3.14159) # Execute the circuit and print the results -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` ### Example: Applying a Toffoli Gate @@ -90,15 +90,15 @@ print(result) qc.create_empty_circuit(3) # Apply the Hadamard gate to the first two qubits -qc.apply_hadamard_gate(0) +qc.apply_hadamard_gate(0) qc.apply_hadamard_gate(1) # Apply the Toffoli gate with qubits 0 and 1 as controls and qubit 2 as target qc.apply_toffoli_gate(0, 1, 2) # Execute the circuit and print the results -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` ### Example: Applying a SWAP Gate @@ -113,8 +113,8 @@ qc.apply_hadamard_gate(0) qc.apply_swap_gate(0, 1) # Execute the circuit and print the results -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` ## 4.4 Visualizing Quantum Circuits @@ -133,7 +133,7 @@ qc.apply_hadamard_gate(0) qc.apply_cnot_gate(0, 1) # Draw the circuit -qc.draw() +qc.draw() ``` -This section introduced the fundamental quantum gates and demonstrated how to apply them using the `qumat` library. In the next section, we will explore how to build more complex quantum circuits by combining these gates. \ No newline at end of file +This section introduced the fundamental quantum gates and demonstrated how to apply them using the `qumat` library. In the next section, we will explore how to build more complex quantum circuits by combining these gates. diff --git a/website/quantum-computing-primer/05_quantum_circuits/index.md b/website/quantum-computing-primer/05_quantum_circuits/index.md index 78da7fe354..fa3a71f738 100644 --- a/website/quantum-computing-primer/05_quantum_circuits/index.md +++ b/website/quantum-computing-primer/05_quantum_circuits/index.md @@ -15,12 +15,12 @@ A quantum circuit is a sequence of quantum gates applied to qubits. The `qumat` To create a quantum circuit with two qubits, we first initialize the circuit and then apply gates to the qubits. -```python +```python from qumat import QuMat # Initialize the quantum circuit with 2 qubits -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply a Hadamard gate to the first qubit @@ -30,8 +30,8 @@ qc.apply_hadamard_gate(0) qc.apply_cnot_gate(0, 1) # Execute the circuit and get the results -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` In this example, we create a quantum circuit with two qubits. We apply a Hadamard gate to the first qubit, which puts it into a superposition state. Then, we apply a CNOT gate, which entangles the two qubits. Finally, we execute the circuit and print the measurement results. @@ -40,12 +40,12 @@ In this example, we create a quantum circuit with two qubits. We apply a Hadamar A Bell state is a specific type of entangled quantum state. Let's create a Bell state using `qumat`. -```python +```python from qumat import QuMat # Initialize the quantum circuit with 2 qubits -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply a Hadamard gate to the first qubit @@ -55,8 +55,8 @@ qc.apply_hadamard_gate(0) qc.apply_cnot_gate(0, 1) # Execute the circuit and get the results -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` This code creates a Bell state by applying a Hadamard gate to the first qubit and then a CNOT gate with the first qubit as the control and the second qubit as the target. The result is an entangled state where the measurement outcomes of the two qubits are correlated. @@ -69,12 +69,12 @@ Visualizing quantum circuits is an essential part of understanding and debugging To visualize a quantum circuit, you can use the `draw` method provided by `qumat`. -```python +```python from qumat import QuMat # Initialize the quantum circuit with 2 qubits -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply a Hadamard gate to the first qubit @@ -84,7 +84,7 @@ qc.apply_hadamard_gate(0) qc.apply_cnot_gate(0, 1) # Draw the circuit -qc.draw() +qc.draw() ``` This code will print a textual representation of the quantum circuit, showing the sequence of gates applied to the qubits. This visualization helps in understanding the structure of the circuit and the flow of quantum information. @@ -95,12 +95,12 @@ Quantum circuits can be made more complex by combining multiple gates. Let's cre ### Example: Creating a Complex Quantum Circuit -```python +```python from qumat import QuMat # Initialize the quantum circuit with 3 qubits -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(3) # Apply a Hadamard gate to the first qubit @@ -113,8 +113,8 @@ qc.apply_cnot_gate(0, 1) qc.apply_toffoli_gate(0, 1, 2) # Execute the circuit and get the results -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` In this example, we create a quantum circuit with three qubits. We apply a Hadamard gate to the first qubit, a CNOT gate with the first qubit as control and the second qubit as target, and a Toffoli gate with the first and second qubits as controls and the third qubit as target. This creates a more complex entangled state. @@ -123,4 +123,4 @@ In this example, we create a quantum circuit with three qubits. We apply a Hadam In this section, we explored how to build and visualize quantum circuits using the `qumat` library. We started with simple circuits and gradually built more complex ones by combining multiple gates. Visualizing these circuits helps in understanding the flow of quantum information and debugging quantum algorithms. -Next, we will dive deeper into quantum entanglement and its applications in quantum computing. \ No newline at end of file +Next, we will dive deeper into quantum entanglement and its applications in quantum computing. diff --git a/website/quantum-computing-primer/06_quantum_entanglement/index.md b/website/quantum-computing-primer/06_quantum_entanglement/index.md index ea0e0dd5c9..fab75177bc 100644 --- a/website/quantum-computing-primer/06_quantum_entanglement/index.md +++ b/website/quantum-computing-primer/06_quantum_entanglement/index.md @@ -22,12 +22,12 @@ In this section, we will explore how to create and measure entangled states usin A Bell state can be created by applying a Hadamard gate to the first qubit, followed by a CNOT gate with the first qubit as the control and the second qubit as the target. This results in a state where the two qubits are perfectly correlated. -```python +```python from qumat import QuMat # Initialize the quantum circuit with 2 qubits -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply a Hadamard gate to the first qubit @@ -37,8 +37,8 @@ qc.apply_hadamard_gate(0) qc.apply_cnot_gate(0, 1) # Execute the circuit and measure the results -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` ### Expected Output: @@ -47,8 +47,8 @@ The output will show the measurement results of the two qubits. Since the qubits ### Visualizing the Circuit: You can also visualize the circuit to better understand the sequence of operations: -```python -qc.draw() +```python +qc.draw() ``` ### Explanation: @@ -63,8 +63,8 @@ Once the qubits are entangled, measuring one qubit will instantly determine the ```python # Execute the circuit and measure the results -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` ### Expected Output: @@ -86,21 +86,21 @@ Quantum teleportation is a protocol that allows the transfer of quantum informat qc.create_empty_circuit(3) # Create an entangled pair between qubit 1 and qubit 2 -qc.apply_hadamard_gate(1) +qc.apply_hadamard_gate(1) qc.apply_cnot_gate(1, 2) # Prepare the qubit to be teleported (qubit 0) qc.apply_hadamard_gate(0) # Perform the teleportation protocol -qc.apply_cnot_gate(0, 1) -qc.apply_hadamard_gate(0) -qc.apply_cnot_gate(1, 2) +qc.apply_cnot_gate(0, 1) +qc.apply_hadamard_gate(0) +qc.apply_cnot_gate(1, 2) qc.apply_toffoli_gate(0, 1, 2) # Measure the qubits -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` ### Expected Output: @@ -108,4 +108,4 @@ The output will show the measurement results, demonstrating that the state of qu ## 6.3 Conclusion -Quantum entanglement is a powerful and essential concept in quantum computing. By understanding how to create and manipulate entangled states using `qumat`, you can begin to explore more advanced quantum algorithms and applications. In the next section, we will delve into quantum algorithms, starting with the Deutsch-Jozsa algorithm. \ No newline at end of file +Quantum entanglement is a powerful and essential concept in quantum computing. By understanding how to create and manipulate entangled states using `qumat`, you can begin to explore more advanced quantum algorithms and applications. In the next section, we will delve into quantum algorithms, starting with the Deutsch-Jozsa algorithm. diff --git a/website/quantum-computing-primer/07_quantum_algorithms/index.md b/website/quantum-computing-primer/07_quantum_algorithms/index.md index 1965af072e..c1479692fd 100644 --- a/website/quantum-computing-primer/07_quantum_algorithms/index.md +++ b/website/quantum-computing-primer/07_quantum_algorithms/index.md @@ -6,39 +6,39 @@ title: Quantum Algorithms # 7. Quantum Algorithms Quantum algorithms leverage the unique properties of quantum mechanics, such as superposition and entanglement, to solve problems more efficiently than classical algorithms. In this section, we will explore two fundamental quantum algorithms: the **Deutsch-Jozsa Algorithm** and **Grover's Algorithm**. We will also provide implementations using the `qumat` library. - ---- + +--- ## 7.1 Deutsch-Jozsa Algorithm -The Deutsch-Jozsa algorithm is one of the earliest quantum algorithms that -demonstrates the potential of quantum computing. It solves a specific problem +The Deutsch-Jozsa algorithm is one of the earliest quantum algorithms that +demonstrates the potential of quantum computing. It solves a specific problem exponentially faster than any classical algorithm. ### Problem Statement -Given a function $ f: \{0,1\}^n \rightarrow \{0,1\} $, determine whether the -function is **constant** (returns the same value for all inputs) or **balanced** +Given a function $ f: \{0,1\}^n \rightarrow \{0,1\} $, determine whether the +function is **constant** (returns the same value for all inputs) or **balanced** (returns 0 for half of the inputs and 1 for the other half). ### Quantum Solution -The Deutsch-Jozsa algorithm uses quantum parallelism to evaluate the function -over all possible inputs simultaneously. It requires only **one query** to the -function, whereas a classical algorithm would need $ 2^{n-1} + 1 $ queries in +The Deutsch-Jozsa algorithm uses quantum parallelism to evaluate the function +over all possible inputs simultaneously. It requires only **one query** to the +function, whereas a classical algorithm would need $ 2^{n-1} + 1 $ queries in the worst case. ### Implementation with `qumat` Here’s how you can implement the Deutsch-Jozsa algorithm using `qumat`: -```python +```python from qumat import QuMat # Initialize the quantum circuit with 2 qubits -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply Hadamard gates to both qubits -qc.apply_hadamard_gate(0) +qc.apply_hadamard_gate(0) qc.apply_hadamard_gate(1) # Apply the oracle (example: constant function) @@ -50,20 +50,20 @@ qc.apply_cnot_gate(0, 1) qc.apply_hadamard_gate(0) # Measure the first qubit -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` ### Explanation - If the function is **constant**, the first qubit will always measure as `0`. - If the function is **balanced**, the first qubit will measure as `1` with high probability. ---- +--- ## 7.2 Grover's Algorithm -Grover's algorithm is a quantum search algorithm that can search an unsorted -database of $ N $ items in $ O(\sqrt{N}) $ time, compared to $ O(N) $ for +Grover's algorithm is a quantum search algorithm that can search an unsorted +database of $ N $ items in $ O(\sqrt{N}) $ time, compared to $ O(N) $ for classical algorithms. ### Problem Statement @@ -77,44 +77,44 @@ Grover's algorithm uses amplitude amplification to increase the probability of m ### Implementation with `qumat` Here’s a simplified implementation of Grover's algorithm using `qumat`: -```python +```python from qumat import QuMat # Initialize the quantum circuit with 3 qubits -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(3) # Apply Hadamard gates to all qubits -qc.apply_hadamard_gate(0) -qc.apply_hadamard_gate(1) +qc.apply_hadamard_gate(0) +qc.apply_hadamard_gate(1) qc.apply_hadamard_gate(2) # Apply the oracle (example: marks the state |110>) -qc.apply_pauli_x_gate(0) -qc.apply_pauli_x_gate(1) -qc.apply_toffoli_gate(0, 1, 2) -qc.apply_pauli_x_gate(0) +qc.apply_pauli_x_gate(0) +qc.apply_pauli_x_gate(1) +qc.apply_toffoli_gate(0, 1, 2) +qc.apply_pauli_x_gate(0) qc.apply_pauli_x_gate(1) # Apply the diffusion operator (Grover's diffusion) -qc.apply_hadamard_gate(0) -qc.apply_hadamard_gate(1) -qc.apply_hadamard_gate(2) -qc.apply_pauli_x_gate(0) -qc.apply_pauli_x_gate(1) -qc.apply_pauli_x_gate(2) -qc.apply_toffoli_gate(0, 1, 2) -qc.apply_pauli_x_gate(0) -qc.apply_pauli_x_gate(1) -qc.apply_pauli_x_gate(2) -qc.apply_hadamard_gate(0) -qc.apply_hadamard_gate(1) +qc.apply_hadamard_gate(0) +qc.apply_hadamard_gate(1) +qc.apply_hadamard_gate(2) +qc.apply_pauli_x_gate(0) +qc.apply_pauli_x_gate(1) +qc.apply_pauli_x_gate(2) +qc.apply_toffoli_gate(0, 1, 2) +qc.apply_pauli_x_gate(0) +qc.apply_pauli_x_gate(1) +qc.apply_pauli_x_gate(2) +qc.apply_hadamard_gate(0) +qc.apply_hadamard_gate(1) qc.apply_hadamard_gate(2) # Measure the qubits -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` ### Explanation @@ -122,7 +122,7 @@ print(result) - The diffusion operator amplifies the probability of measuring the marked state. - After running the algorithm, the marked state will have a higher probability of being measured. ---- +--- ## 7.3 Applications of Quantum Algorithms @@ -132,7 +132,7 @@ Quantum algorithms like Deutsch-Jozsa and Grover's are foundational to many adva - **Machine Learning**: Speeding up training and inference in quantum machine learning models. By mastering these algorithms with `qumat`, you can begin to explore the vast potential of quantum computing in real-world applications. - ---- -This section provides a hands-on introduction to quantum algorithms using `qumat`. Experiment with the provided code examples to deepen your understanding of quantum computing principles! \ No newline at end of file +--- + +This section provides a hands-on introduction to quantum algorithms using `qumat`. Experiment with the provided code examples to deepen your understanding of quantum computing principles! diff --git a/website/quantum-computing-primer/08_quantum_error_correction/index.md b/website/quantum-computing-primer/08_quantum_error_correction/index.md index d1bf7ce407..8e32756b40 100644 --- a/website/quantum-computing-primer/08_quantum_error_correction/index.md +++ b/website/quantum-computing-primer/08_quantum_error_correction/index.md @@ -21,30 +21,30 @@ Quantum bits (qubits) are highly susceptible to errors caused by environmental n ### Example: Simple Bit-Flip Error Correction The following example demonstrates a simple bit-flip error correction circuit using `qumat`. The circuit encodes one logical qubit into three physical qubits and corrects a single bit-flip error. -```python +```python from qumat import QuMat # Initialize the quantum circuit with 3 qubits -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(3) # Encode the logical qubit into 3 physical qubits -qc.apply_hadamard_gate(0) -qc.apply_cnot_gate(0, 1) +qc.apply_hadamard_gate(0) +qc.apply_cnot_gate(0, 1) qc.apply_cnot_gate(0, 2) # Simulate a bit-flip error on qubit 1 qc.apply_pauli_x_gate(1) # Error correction steps -qc.apply_cnot_gate(0, 1) -qc.apply_cnot_gate(0, 2) +qc.apply_cnot_gate(0, 1) +qc.apply_cnot_gate(0, 2) qc.apply_toffoli_gate(1, 2, 0) # Execute the circuit and print the results -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` ### Explanation @@ -55,8 +55,8 @@ print(result) ### Visualizing the Circuit You can visualize the error correction circuit using the `draw` method: -```python -qc.draw() +```python +qc.draw() ``` This will display the circuit diagram, showing the encoding, error simulation, and correction steps. @@ -75,7 +75,7 @@ While the above example demonstrates a simple bit-flip error correction, `qumat` ## 8.4 Conclusion Quantum error correction is essential for building reliable quantum computers. By using `qumat`, you can implement and experiment with various error correction techniques, from simple bit-flip correction to more advanced codes like the Shor and Steane codes. As quantum hardware continues to improve, these techniques will play a critical role in realizing the full potential of quantum computing. - ---- -This section provides a foundational understanding of quantum error correction and demonstrates how to implement basic error correction circuits using `qumat`. For further exploration, consider experimenting with more complex error correction codes and their applications in quantum computing. \ No newline at end of file +--- + +This section provides a foundational understanding of quantum error correction and demonstrates how to implement basic error correction circuits using `qumat`. For further exploration, consider experimenting with more complex error correction codes and their applications in quantum computing. diff --git a/website/quantum-computing-primer/09_applications/index.md b/website/quantum-computing-primer/09_applications/index.md index a41cf9047d..211b171941 100644 --- a/website/quantum-computing-primer/09_applications/index.md +++ b/website/quantum-computing-primer/09_applications/index.md @@ -15,16 +15,16 @@ Quantum cryptography leverages the principles of quantum mechanics to create sec ### Example: Implementing a Simple QKD Protocol with `qumat` Below is a simplified example of how to implement a basic QKD protocol using `qumat`. This example demonstrates the generation of a shared key between two parties, Alice and Bob. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Alice prepares her qubits -qc.apply_hadamard_gate(0) +qc.apply_hadamard_gate(0) qc.apply_cnot_gate(0, 1) # Alice sends the second qubit to Bob @@ -32,7 +32,7 @@ qc.apply_cnot_gate(0, 1) qc.apply_hadamard_gate(1) # Execute the circuit and get the results -result = qc.execute_circuit() +result = qc.execute_circuit() print("Shared key:", result) ``` @@ -44,21 +44,21 @@ Quantum simulation involves using a quantum computer to simulate quantum systems ### Example: Simulating a Simple Quantum System with `qumat` In this example, we simulate a simple quantum system, such as a hydrogen molecule, using `qumat`. The goal is to find the ground state energy of the molecule. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'statevector_simulator', 'shots': 1}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'statevector_simulator', 'shots': 1}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply gates to simulate the hydrogen molecule -qc.apply_hadamard_gate(0) -qc.apply_cnot_gate(0, 1) +qc.apply_hadamard_gate(0) +qc.apply_cnot_gate(0, 1) qc.apply_rz_gate(1, 0.5) # Example of a parameterized gate # Execute the circuit and get the final state vector -state_vector = qc.get_final_state_vector() +state_vector = qc.get_final_state_vector() print("Final state vector:", state_vector) ``` ## 9.3 Quantum Machine Learning @@ -69,21 +69,21 @@ Quantum machine learning (QML) is an emerging field that combines quantum comput ### Example: Implementing a Basic Quantum Classifier with `qumat` In this example, we implement a basic quantum classifier using `qumat`. The classifier is trained to distinguish between two classes based on a simple dataset. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply gates to create a quantum classifier -qc.apply_hadamard_gate(0) -qc.apply_cnot_gate(0, 1) +qc.apply_hadamard_gate(0) +qc.apply_cnot_gate(0, 1) qc.apply_ry_gate(1, 0.3) # Example of a parameterized gate # Execute the circuit and get the results -result = qc.execute_circuit() +result = qc.execute_circuit() print("Classification result:", result) ``` @@ -95,22 +95,22 @@ Quantum optimization involves using quantum algorithms to solve optimization pro ### Example: Solving an Optimization Problem with `qumat` In this example, we use `qumat` to implement a simple QAOA circuit to solve a basic optimization problem. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply gates to implement QAOA -qc.apply_hadamard_gate(0) -qc.apply_hadamard_gate(1) -qc.apply_rx_gate(0, 0.5) # Example of a parameterized gate +qc.apply_hadamard_gate(0) +qc.apply_hadamard_gate(1) +qc.apply_rx_gate(0, 0.5) # Example of a parameterized gate qc.apply_ry_gate(1, 0.5) # Example of a parameterized gate # Execute the circuit and get the results -result = qc.execute_circuit() +result = qc.execute_circuit() print("Optimization result:", result) ``` @@ -122,21 +122,21 @@ Quantum chemistry involves the application of quantum mechanics to chemical syst ### Example: Simulating a Chemical Reaction with `qumat` In this example, we use `qumat` to simulate a simple chemical reaction, such as the formation of a hydrogen molecule from two hydrogen atoms. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'statevector_simulator', 'shots': 1}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'statevector_simulator', 'shots': 1}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply gates to simulate the chemical reaction -qc.apply_hadamard_gate(0) -qc.apply_cnot_gate(0, 1) +qc.apply_hadamard_gate(0) +qc.apply_cnot_gate(0, 1) qc.apply_rz_gate(1, 0.5) # Example of a parameterized gate # Execute the circuit and get the final state vector -state_vector = qc.get_final_state_vector() +state_vector = qc.get_final_state_vector() print("Final state vector:", state_vector) ``` @@ -148,21 +148,21 @@ Quantum finance involves the application of quantum computing to financial probl ### Example: Portfolio Optimization with `qumat` In this example, we use `qumat` to implement a simple quantum algorithm for portfolio optimization. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply gates to implement portfolio optimization -qc.apply_hadamard_gate(0) -qc.apply_cnot_gate(0, 1) +qc.apply_hadamard_gate(0) +qc.apply_cnot_gate(0, 1) qc.apply_ry_gate(1, 0.5) # Example of a parameterized gate # Execute the circuit and get the results -result = qc.execute_circuit() +result = qc.execute_circuit() print("Portfolio optimization result:", result) ``` ## 9.7 Quantum Artificial Intelligence @@ -173,21 +173,21 @@ Quantum artificial intelligence (QAI) combines quantum computing with artificial ### Example: Implementing a Quantum Neural Network with `qumat` In this example, we use `qumat` to implement a simple quantum neural network (QNN) for a basic classification task. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply gates to implement a quantum neural network -qc.apply_hadamard_gate(0) -qc.apply_cnot_gate(0, 1) +qc.apply_hadamard_gate(0) +qc.apply_cnot_gate(0, 1) qc.apply_ry_gate(1, 0.5) # Example of a parameterized gate # Execute the circuit and get the results -result = qc.execute_circuit() +result = qc.execute_circuit() print("QNN classification result:", result) ``` @@ -199,21 +199,21 @@ Quantum sensing involves using quantum systems to measure physical quantities wi ### Example: Implementing a Quantum Sensor with `qumat` In this example, we use `qumat` to implement a simple quantum sensor for measuring a physical quantity, such as magnetic field strength. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'statevector_simulator', 'shots': 1}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'statevector_simulator', 'shots': 1}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply gates to implement a quantum sensor -qc.apply_hadamard_gate(0) -qc.apply_cnot_gate(0, 1) +qc.apply_hadamard_gate(0) +qc.apply_cnot_gate(0, 1) qc.apply_rz_gate(1, 0.5) # Example of a parameterized gate # Execute the circuit and get the final state vector -state_vector = qc.get_final_state_vector() +state_vector = qc.get_final_state_vector() print("Quantum sensor measurement:", state_vector) ``` ## 9.9 Quantum Communication @@ -224,21 +224,21 @@ Quantum communication involves the transmission of information using quantum sta ### Example: Implementing Quantum Teleportation with `qumat` In this example, we use `qumat` to implement a quantum teleportation protocol, which allows the transfer of quantum information from one qubit to another. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'statevector_simulator', 'shots': 1}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'statevector_simulator', 'shots': 1}} +qc = QuMat(backend_config) qc.create_empty_circuit(3) # Apply gates to implement quantum teleportation -qc.apply_hadamard_gate(0) -qc.apply_cnot_gate(0, 1) +qc.apply_hadamard_gate(0) +qc.apply_cnot_gate(0, 1) qc.apply_cnot_gate(1, 2) # Execute the circuit and get the final state vector -state_vector = qc.get_final_state_vector() +state_vector = qc.get_final_state_vector() print("Quantum teleportation result:", state_vector) ``` ## 9.10 Quantum Error Correction @@ -249,29 +249,29 @@ Quantum error correction is essential for building reliable quantum computers. Q ### Example: Implementing a Quantum Error Correction Code with `qumat` In this example, we use `qumat` to implement a simple quantum error correction code, such as the bit-flip code. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(3) # Apply gates to implement the bit-flip code -qc.apply_hadamard_gate(0) -qc.apply_cnot_gate(0, 1) +qc.apply_hadamard_gate(0) +qc.apply_cnot_gate(0, 1) qc.apply_cnot_gate(0, 2) # Simulate an error (e.g., bit flip on qubit 1) qc.apply_pauli_x_gate(1) # Error correction steps -qc.apply_cnot_gate(0, 1) -qc.apply_cnot_gate(0, 2) +qc.apply_cnot_gate(0, 1) +qc.apply_cnot_gate(0, 2) qc.apply_toffoli_gate(1, 2, 0) # Execute the circuit and get the results -result = qc.execute_circuit() +result = qc.execute_circuit() print("Error correction result:", result) ``` ## 9.11 Quantum Games @@ -282,20 +282,20 @@ Quantum games are games that incorporate quantum mechanics into their rules or s ### Example: Implementing a Quantum Game with `qumat` In this example, we use `qumat` to implement a simple quantum game, such as the quantum version of the classic game "Rock-Paper-Scissors." -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply gates to implement the quantum game -qc.apply_hadamard_gate(0) +qc.apply_hadamard_gate(0) qc.apply_cnot_gate(0, 1) # Execute the circuit and get the results -result = qc.execute_circuit() +result = qc.execute_circuit() print("Quantum game result:", result) ``` ## 9.12 Quantum Random Number Generation @@ -306,19 +306,19 @@ Quantum random number generation (QRNG) uses the inherent randomness of quantum ### Example: Implementing a Quantum Random Number Generator with `qumat` In this example, we use `qumat` to implement a simple quantum random number generator. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(1) # Apply a Hadamard gate to generate a random bit qc.apply_hadamard_gate(0) # Execute the circuit and get the results -result = qc.execute_circuit() +result = qc.execute_circuit() print("Random number:", result) ``` ## 9.13 Quantum Image Processing @@ -329,20 +329,20 @@ Quantum image processing involves using quantum algorithms to process and analyz ### Example: Implementing a Quantum Image Processing Algorithm with `qumat` In this example, we use `qumat` to implement a simple quantum image processing algorithm, such as edge detection. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply gates to implement edge detection -qc.apply_hadamard_gate(0) +qc.apply_hadamard_gate(0) qc.apply_cnot_gate(0, 1) # Execute the circuit and get the results -result = qc.execute_circuit() +result = qc.execute_circuit() print("Edge detection result:", result) ``` ## 9.14 Quantum Natural Language Processing @@ -353,20 +353,20 @@ Quantum natural language processing (QNLP) involves using quantum algorithms to ### Example: Implementing a Quantum NLP Algorithm with `qumat` In this example, we use `qumat` to implement a simple quantum NLP algorithm, such as text classification. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply gates to implement text classification -qc.apply_hadamard_gate(0) +qc.apply_hadamard_gate(0) qc.apply_cnot_gate(0, 1) # Execute the circuit and get the results -result = qc.execute_circuit() +result = qc.execute_circuit() print("Text classification result:", result) ``` ## 9.15 Quantum Robotics @@ -377,20 +377,20 @@ Quantum robotics involves using quantum computing to enhance the capabilities of ### Example: Implementing a Quantum Robotics Algorithm with `qumat` In this example, we use `qumat` to implement a simple quantum robotics algorithm, such as path planning. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply gates to implement path planning -qc.apply_hadamard_gate(0) +qc.apply_hadamard_gate(0) qc.apply_cnot_gate(0, 1) # Execute the circuit and get the results -result = qc.execute_circuit() +result = qc.execute_circuit() print("Path planning result:", result) ``` ## 9.16 Quantum Internet @@ -401,20 +401,20 @@ The quantum internet is a proposed network that uses quantum communication proto ### Example: Implementing a Quantum Internet Protocol with `qumat` In this example, we use `qumat` to implement a simple quantum internet protocol, such as quantum key distribution. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply gates to implement quantum key distribution -qc.apply_hadamard_gate(0) +qc.apply_hadamard_gate(0) qc.apply_cnot_gate(0, 1) # Execute the circuit and get the results -result = qc.execute_circuit() +result = qc.execute_circuit() print("Quantum key distribution result:", result) ``` ## 9.17 Quantum Biology @@ -425,20 +425,20 @@ Quantum biology explores the role of quantum mechanics in biological processes. ### Example: Simulating a Biological Process with `qumat` In this example, we use `qumat` to simulate a simple biological process, such as photosynthesis. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'statevector_simulator', 'shots': 1}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'statevector_simulator', 'shots': 1}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply gates to simulate photosynthesis -qc.apply_hadamard_gate(0) +qc.apply_hadamard_gate(0) qc.apply_cnot_gate(0, 1) # Execute the circuit and get the final state vector -state_vector = qc.get_final_state_vector() +state_vector = qc.get_final_state_vector() print("Photosynthesis simulation result:", state_vector) ``` ## 9.18 Quantum Materials Science @@ -449,14 +449,14 @@ Quantum materials science involves using quantum computing to study and design n ### Example: Simulating a Material with `qumat` In this example, we use `qumat` to simulate a simple material, such as graphene. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'statevector_simulator', 'shots': 1}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'statevector_simulator', 'shots': 1}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply gates to simulate graphene qc.apply_hadamard_gate(0 -``` \ No newline at end of file +``` diff --git a/website/quantum-computing-primer/10_advanced_topics/index.md b/website/quantum-computing-primer/10_advanced_topics/index.md index c0cd49fa23..d8695062f9 100644 --- a/website/quantum-computing-primer/10_advanced_topics/index.md +++ b/website/quantum-computing-primer/10_advanced_topics/index.md @@ -15,27 +15,27 @@ The Quantum Fourier Transform (QFT) is a quantum analogue of the classical Fouri ### Implementation with `qumat` Below is an example of how to implement the QFT using `qumat`. This example assumes a 3-qubit system. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(3) # Apply the Quantum Fourier Transform -def apply_qft(qc, n_qubits): - for qubit in range(n_qubits): - qc.apply_hadamard_gate(qubit) - for next_qubit in range(qubit + 1, n_qubits): - angle = 2 * 3.14159 / (2 ** (next_qubit - qubit + 1)) +def apply_qft(qc, n_qubits): + for qubit in range(n_qubits): + qc.apply_hadamard_gate(qubit) + for next_qubit in range(qubit + 1, n_qubits): + angle = 2 * 3.14159 / (2 ** (next_qubit - qubit + 1)) qc.apply_cu_gate(next_qubit, qubit, angle) apply_qft(qc, 3) # Execute the circuit and print the results -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` ## 10.2 Quantum Phase Estimation (QPE) @@ -46,29 +46,29 @@ Quantum Phase Estimation (QPE) is a quantum algorithm used to estimate the phase ### Implementation with `qumat` Below is an example of how to implement QPE using `qumat`. This example assumes a 3-qubit system and a simple unitary operator. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(3) # Apply the Quantum Phase Estimation -def apply_qpe(qc, n_qubits): - for qubit in range(n_qubits): - qc.apply_hadamard_gate(qubit) - # Apply controlled unitary operations (simplified example) - qc.apply_cu_gate(1, 0, 3.14159 / 2) - qc.apply_cu_gate(2, 1, 3.14159 / 4) -# Inverse QFT +def apply_qpe(qc, n_qubits): + for qubit in range(n_qubits): + qc.apply_hadamard_gate(qubit) + # Apply controlled unitary operations (simplified example) + qc.apply_cu_gate(1, 0, 3.14159 / 2) + qc.apply_cu_gate(2, 1, 3.14159 / 4) +# Inverse QFT apply_qft(qc, n_qubits) apply_qpe(qc, 3) # Execute the circuit and print the results -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` ## 10.3 Quantum Annealing @@ -79,32 +79,32 @@ Quantum Annealing is a quantum computing technique used to solve optimization pr ### Implementation with `qumat` Below is an example of how to implement a simple quantum annealing process using `qumat`. This example assumes a 2-qubit system and a simple objective function. -```python +```python from qumat import QuMat # Initialize the quantum circuit -backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} -qc = QuMat(backend_config) +backend_config = {'backend_name': 'qiskit', 'backend_options': {'simulator_type': 'qasm_simulator', 'shots': 1000}} +qc = QuMat(backend_config) qc.create_empty_circuit(2) # Apply the Quantum Annealing process -def apply_quantum_annealing(qc, n_qubits): - for qubit in range(n_qubits): - qc.apply_hadamard_gate(qubit) - # Apply a simple Hamiltonian (simplified example) - qc.apply_rx_gate(0, 3.14159 / 2) - qc.apply_ry_gate(1, 3.14159 / 2) - # Measure the qubits +def apply_quantum_annealing(qc, n_qubits): + for qubit in range(n_qubits): + qc.apply_hadamard_gate(qubit) + # Apply a simple Hamiltonian (simplified example) + qc.apply_rx_gate(0, 3.14159 / 2) + qc.apply_ry_gate(1, 3.14159 / 2) + # Measure the qubits qc.execute_circuit() apply_quantum_annealing(qc, 2) # Execute the circuit and print the results -result = qc.execute_circuit() -print(result) +result = qc.execute_circuit() +print(result) ``` ## Conclusion In this section, we explored advanced topics in quantum computing, including the Quantum Fourier Transform, Quantum Phase Estimation, and Quantum Annealing. Each topic was accompanied by a practical example using the `qumat` library. These advanced techniques are essential for understanding and implementing more complex quantum algorithms and applications. -For further reading, consider exploring the official documentation of `qumat` and other quantum computing resources to deepen your understanding of these topics. \ No newline at end of file +For further reading, consider exploring the official documentation of `qumat` and other quantum computing resources to deepen your understanding of these topics. diff --git a/website/quantum-computing-primer/index.md b/website/quantum-computing-primer/index.md index fcadd1c4af..4bef30f48d 100644 --- a/website/quantum-computing-primer/index.md +++ b/website/quantum-computing-primer/index.md @@ -7,7 +7,7 @@ title: Quantum Computing Primer

    Mahout's new quantum computing layer for building ML circuits on simulators and real quantum hardware.

    Explore Qumat → - + # Outline for an In-Depth Primer in Quantum Computing Using `qumat` @@ -54,4 +54,3 @@ title: Quantum Computing Primer - **10.1 Quantum Fourier Transform** - **10.2 Quantum Phase Estimation** - **10.3 Quantum Annealing** -