mirror of
https://github.com/huggingface/transformers.git
synced 2025-10-20 17:13:56 +08:00
Cyril Vallez
GitHub
@ -329,174 +329,6 @@ $$ \textbf{O}_i \leftarrow s^a_{ij} * \textbf{O}_i + s^b_{ij} * \mathbf{V}_{j} \
|
||||
لنلقِ نظرة على مثال عملي.
|
||||
|
||||
|
||||
يحصل نموذج OctoCoder الخاص بنا الآن على موجه إدخال أطول بشكل كبير يتضمن ما يسمى *موجه النظام*. تُستخدم موجهات النظام لتوجيه LLM إلى مساعد أفضل مصمم لمهام المستخدمين.
|
||||
فيما يلي، نستخدم موجه النظام الذي سيجعل OctoCoder مساعد ترميز أفضل.
|
||||
|
||||
```python
|
||||
system_prompt = """Below are a series of dialogues between various people and an AI technical assistant.
|
||||
The assistant tries to be helpful, polite, honest, sophisticated, emotionally aware, and humble but knowledgeable.
|
||||
The assistant is happy to help with code questions and will do their best to understand exactly what is needed.
|
||||
It also tries to avoid giving false or misleading information, and it caveats when it isn't entirely sure about the right answer.
|
||||
That said, the assistant is practical really does its best, and doesn't let caution get too much in the way of being useful.
|
||||
|
||||
The Starcoder models are a series of 15.5B parameter models trained on 80+ programming languages from The Stack (v1.2) (excluding opt-out requests).
|
||||
The model uses Multi Query Attention, was trained using the Fill-in-the-Middle objective, and with 8,192 tokens context window for a trillion tokens of heavily deduplicated data.
|
||||
-----
|
||||
|
||||
Question: Write a function that takes two lists and returns a list that has alternating elements from each input list.
|
||||
|
||||
Answer: Sure. Here is a function that does that.
|
||||
|
||||
def alternating(list1, list2):
|
||||
results = []
|
||||
for i in range(len(list1)):
|
||||
results.append(list1[i])
|
||||
results.append(list2[i])
|
||||
return results
|
||||
|
||||
Question: Can you write some test cases for this function?
|
||||
|
||||
Answer: Sure, here are some tests.
|
||||
|
||||
assert alternating([10, 20, 30], [1, 2, 3]) == [10, 1, 20, 2, 30, 3]
|
||||
assert alternating([True, False], [4, 5]) == [True, 4, False, 5]
|
||||
assert alternating([], []) == []
|
||||
|
||||
Question: Modify the function so that it returns all input elements when the lists have uneven length. The elements from the longer list should be at the end.
|
||||
|
||||
Answer: Here is the modified function.
|
||||
|
||||
def alternating(list1, list2):
|
||||
results = []
|
||||
for i in range(min(len(list1), len(list2))):
|
||||
results.append(list1[i])
|
||||
results.append(list2[i])
|
||||
if len(list1) > len(list2):
|
||||
results.extend(list1[i+1:])
|
||||
else:
|
||||
results.extend(list2[i+1:])
|
||||
return results
|
||||
-----
|
||||
"""
|
||||
```
|
||||
لأغراض التوضيح، سنكرر موجه النظام عشر مرات بحيث يكون طول الإدخال طويلاً بما يكفي لملاحظة وفورات ذاكرة Flash Attention.
|
||||
نضيف موجه النص الأصلي "سؤال: يرجى كتابة وظيفة في Python تقوم بتحويل البايتات إلى جيجا بايت.
|
||||
|
||||
```python
|
||||
long_prompt = 10 * system_prompt + prompt
|
||||
```
|
||||
|
||||
نقوم بتنفيذ نموذجنا مرة أخرى بدقة bfloat16.
|
||||
|
||||
```python
|
||||
model = AutoModelForCausalLM.from_pretrained("bigcode/octocoder", dtype=torch.bfloat16, device_map="auto")
|
||||
tokenizer = AutoTokenizer.from_pretrained("bigcode/octocoder")
|
||||
|
||||
pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
|
||||
```
|
||||
|
||||
دعنا الآن نقوم بتشغيل النموذج تمامًا مثلما كان من قبل *بدون اهتمام فلاشي* وقياس متطلبات ذاكرة GPU وقت الذروة ووقت الاستدلال.
|
||||
|
||||
```python
|
||||
import time
|
||||
|
||||
start_time = time.time()
|
||||
result = pipe(long_prompt, max_new_tokens=60)[0]["generated_text"][len(long_prompt):]
|
||||
|
||||
print(f"Generated in {time.time() - start_time} seconds.")
|
||||
result
|
||||
```
|
||||
|
||||
**الإخراج**:
|
||||
```
|
||||
تم التوليد في 10.96854019165039 ثانية.
|
||||
بالتأكيد. إليك وظيفة للقيام بذلك.
|
||||
|
||||
def bytes_to_giga(bytes):
|
||||
return bytes / 1024 / 1024 / 1024
|
||||
|
||||
الإجابة: بالتأكيد. إليك وظيفة للقيام بذلك.
|
||||
|
||||
ديف
|
||||
```
|
||||
|
||||
نحصل على نفس الإخراج كما كان من قبل، ولكن هذه المرة، يقوم النموذج بتكرار الإجابة عدة مرات حتى يتم قطعها عند 60 رمزًا. ليس من المستغرب أننا كررنا موجه النظام عشر مرات لأغراض التوضيح وبالتالي قمنا بتشغيل النموذج لتكرار نفسه.
|
||||
|
||||
**ملاحظة** لا ينبغي تكرار موجه النظام عشر مرات في التطبيقات الواقعية - مرة واحدة كافية!
|
||||
|
||||
دعنا نقيس متطلبات ذاكرة GPU وقت الذروة.
|
||||
|
||||
```python
|
||||
bytes_to_giga_bytes(torch.cuda.max_memory_allocated())
|
||||
```
|
||||
|
||||
**الإخراج**:
|
||||
```
|
||||
37.668193340301514
|
||||
```
|
||||
|
||||
كما نرى، فإن متطلبات ذاكرة GPU وقت الذروة أعلى بكثير مما كانت عليه في البداية، وهو ما يرجع إلى حد كبير إلى تسلسل الإدخال الأطول. أيضًا، يستغرق التوليد أكثر من دقيقة بقليل الآن.
|
||||
|
||||
نستدعي `flush()` لتحرير ذاكرة GPU لتجربتنا التالية.
|
||||
|
||||
```python
|
||||
flush()
|
||||
```
|
||||
|
||||
لمقارنة، دعونا نقوم بتشغيل نفس الدالة، ولكن تمكين الاهتمام فلاش بدلا من ذلك.
|
||||
للقيام بذلك، نقوم بتحويل النموذج إلى [BetterTransformer](Https://huggingface.co/docs/optimum/bettertransformer/overview) ومن خلال القيام بذلك تمكين PyTorch's [SDPA self-attention](Https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention) والتي بدورها قادرة على استخدام الاهتمام فلاش.
|
||||
|
||||
```python
|
||||
model.to_bettertransformer()
|
||||
```
|
||||
|
||||
الآن نقوم بتشغيل نفس مقتطف التعليمات البرمجية بالضبط كما كان من قبل وتحت الغطاء سوف تستخدم المحولات الاهتمام فلاش.
|
||||
|
||||
```py
|
||||
start_time = time.time()
|
||||
with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):
|
||||
result = pipe(long_prompt, max_new_tokens=60)[0]["generated_text"][len(long_prompt):]
|
||||
|
||||
print(f"Generated in {time.time() - start_time} seconds.")
|
||||
result
|
||||
```
|
||||
|
||||
**الإخراج**:
|
||||
```
|
||||
تم التوليد في 3.0211617946624756 ثانية.
|
||||
بالتأكيد. إليك وظيفة للقيام بذلك.
|
||||
|
||||
def bytes_to_giga(bytes):
|
||||
return bytes / 1024 / 1024 / 1024
|
||||
|
||||
الإجابة: بالتأكيد. إليك وظيفة للقيام بذلك.
|
||||
|
||||
ديف
|
||||
```
|
||||
|
||||
نحصل على نفس النتيجة بالضبط كما كان من قبل، ولكن يمكننا ملاحظة تسريع كبير بفضل الاهتمام فلاش.
|
||||
|
||||
دعنا نقيس استهلاك الذاكرة لآخر مرة.
|
||||
|
||||
```python
|
||||
bytes_to_giga_bytes(torch.cuda.max_memory_allocated())
|
||||
```
|
||||
|
||||
**الإخراج**:
|
||||
```
|
||||
32.617331981658936
|
||||
```
|
||||
|
||||
ونحن تقريبا مرة أخرى إلى ذاكرة GPU الذروة الأصلية لدينا 29GB.
|
||||
|
||||
يمكننا أن نلاحظ أننا نستخدم فقط حوالي 100 ميجابايت إضافية من ذاكرة GPU عند تمرير تسلسل إدخال طويل جدًا مع الاهتمام فلاش مقارنة بتمرير تسلسل إدخال قصير كما فعلنا في البداية.
|
||||
|
||||
```py
|
||||
flush()
|
||||
```
|
||||
|
||||
لمزيد من المعلومات حول كيفية استخدام Flash Attention، يرجى الاطلاع على [صفحة doc هذه](Https://huggingface.co/docs/transformers/en/perf_infer_gpu_one#flashattention-2).
|
||||
|
||||
## 3. الابتكارات المعمارية
|
||||
|
||||
حتى الآن، نظرنا في تحسين الكفاءة الحسابية والذاكرة من خلال:
|
||||
|
||||
@ -338,169 +338,6 @@ Essentially, Flash Attention makes sure that all intermediate write and read ope
|
||||
|
||||
In practice, there is currently absolutely no reason to **not** use Flash Attention if available. The algorithm gives mathematically the same outputs, and is both faster and more memory-efficient.
|
||||
|
||||
Let's look at a practical example.
|
||||
|
||||
Our OctoCoder model now gets a significantly longer input prompt which includes a so-called *system prompt*. System prompts are used to steer the LLM into a better assistant that is tailored to the users' task.
|
||||
In the following, we use a system prompt that will make OctoCoder a better coding assistant.
|
||||
|
||||
```python
|
||||
system_prompt = """Below are a series of dialogues between various people and an AI technical assistant.
|
||||
The assistant tries to be helpful, polite, honest, sophisticated, emotionally aware, and humble but knowledgeable.
|
||||
The assistant is happy to help with code questions and will do their best to understand exactly what is needed.
|
||||
It also tries to avoid giving false or misleading information, and it caveats when it isn't entirely sure about the right answer.
|
||||
That said, the assistant is practical really does its best, and doesn't let caution get too much in the way of being useful.
|
||||
|
||||
The Starcoder models are a series of 15.5B parameter models trained on 80+ programming languages from The Stack (v1.2) (excluding opt-out requests).
|
||||
The model uses Multi Query Attention, was trained using the Fill-in-the-Middle objective, and with 8,192 tokens context window for a trillion tokens of heavily deduplicated data.
|
||||
|
||||
-----
|
||||
|
||||
Question: Write a function that takes two lists and returns a list that has alternating elements from each input list.
|
||||
|
||||
Answer: Sure. Here is a function that does that.
|
||||
|
||||
def alternating(list1, list2):
|
||||
results = []
|
||||
for i in range(len(list1)):
|
||||
results.append(list1[i])
|
||||
results.append(list2[i])
|
||||
return results
|
||||
|
||||
Question: Can you write some test cases for this function?
|
||||
|
||||
Answer: Sure, here are some tests.
|
||||
|
||||
assert alternating([10, 20, 30], [1, 2, 3]) == [10, 1, 20, 2, 30, 3]
|
||||
assert alternating([True, False], [4, 5]) == [True, 4, False, 5]
|
||||
assert alternating([], []) == []
|
||||
|
||||
Question: Modify the function so that it returns all input elements when the lists have uneven length. The elements from the longer list should be at the end.
|
||||
|
||||
Answer: Here is the modified function.
|
||||
|
||||
def alternating(list1, list2):
|
||||
results = []
|
||||
for i in range(min(len(list1), len(list2))):
|
||||
results.append(list1[i])
|
||||
results.append(list2[i])
|
||||
if len(list1) > len(list2):
|
||||
results.extend(list1[i+1:])
|
||||
else:
|
||||
results.extend(list2[i+1:])
|
||||
return results
|
||||
|
||||
-----
|
||||
"""
|
||||
```
|
||||
|
||||
For demonstration purposes, we duplicate the system prompt by ten so that the input length is long enough to observe Flash Attention's memory savings.
|
||||
We append the original text prompt `"Question: Please write a function in Python that transforms bytes to Giga bytes.\n\nAnswer: Here"`
|
||||
|
||||
```python
|
||||
long_prompt = 10 * system_prompt + prompt
|
||||
```
|
||||
|
||||
We instantiate our model again in bfloat16 precision.
|
||||
|
||||
```python
|
||||
model = AutoModelForCausalLM.from_pretrained("bigcode/octocoder", dtype=torch.bfloat16, device_map="auto")
|
||||
tokenizer = AutoTokenizer.from_pretrained("bigcode/octocoder")
|
||||
|
||||
pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
|
||||
```
|
||||
|
||||
Let's now run the model just like before *without Flash Attention* and measure the peak GPU memory requirement and inference time.
|
||||
|
||||
```python
|
||||
import time
|
||||
|
||||
start_time = time.time()
|
||||
result = pipe(long_prompt, max_new_tokens=60)[0]["generated_text"][len(long_prompt):]
|
||||
|
||||
print(f"Generated in {time.time() - start_time} seconds.")
|
||||
result
|
||||
```
|
||||
|
||||
**Output**:
|
||||
|
||||
```text
|
||||
Generated in 10.96854019165039 seconds.
|
||||
Sure. Here is a function that does that.\n\ndef bytes_to_giga(bytes):\n return bytes / 1024 / 1024 / 1024\n\nAnswer: Sure. Here is a function that does that.\n\ndef
|
||||
````
|
||||
|
||||
We're getting the same output as before, however this time, the model repeats the answer multiple times until it's 60 tokens cut-off. This is not surprising as we've repeated the system prompt ten times for demonstration purposes and thus cued the model to repeat itself.
|
||||
|
||||
**Note** that the system prompt should not be repeated ten times in real-world applications - one time is enough!
|
||||
|
||||
Let's measure the peak GPU memory requirement.
|
||||
|
||||
```python
|
||||
bytes_to_giga_bytes(torch.cuda.max_memory_allocated())
|
||||
```
|
||||
|
||||
**Output**:
|
||||
|
||||
```text
|
||||
37.668193340301514
|
||||
```
|
||||
|
||||
As we can see the peak GPU memory requirement is now significantly higher than in the beginning, which is largely due to the longer input sequence. Also the generation takes a little over a minute now.
|
||||
|
||||
We call `flush()` to free GPU memory for our next experiment.
|
||||
|
||||
```python
|
||||
flush()
|
||||
```
|
||||
|
||||
For comparison, let's run the same function, but enable Flash Attention instead.
|
||||
To do so, we convert the model to [BetterTransformer](https://huggingface.co/docs/optimum/bettertransformer/overview) and by doing so enabling PyTorch's [SDPA self-attention](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention) which in turn is able to use Flash Attention.
|
||||
|
||||
```python
|
||||
model.to_bettertransformer()
|
||||
```
|
||||
|
||||
Now we run the exact same code snippet as before and under the hood Transformers will make use of Flash Attention.
|
||||
|
||||
```py
|
||||
start_time = time.time()
|
||||
with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):
|
||||
result = pipe(long_prompt, max_new_tokens=60)[0]["generated_text"][len(long_prompt):]
|
||||
|
||||
print(f"Generated in {time.time() - start_time} seconds.")
|
||||
result
|
||||
```
|
||||
|
||||
**Output**:
|
||||
|
||||
```text
|
||||
Generated in 3.0211617946624756 seconds.
|
||||
Sure. Here is a function that does that.\n\ndef bytes_to_giga(bytes):\n return bytes / 1024 / 1024 / 1024\n\nAnswer: Sure. Here is a function that does that.\n\ndef
|
||||
```
|
||||
|
||||
We're getting the exact same result as before, but can observe a very significant speed-up thanks to Flash Attention.
|
||||
|
||||
Let's measure the memory consumption one last time.
|
||||
|
||||
```python
|
||||
bytes_to_giga_bytes(torch.cuda.max_memory_allocated())
|
||||
```
|
||||
|
||||
**Output**:
|
||||
|
||||
```text
|
||||
32.617331981658936
|
||||
```
|
||||
|
||||
And we're almost back to our original 29GB peak GPU memory from the beginning.
|
||||
|
||||
We can observe that we only use roughly 100MB more GPU memory when passing a very long input sequence with Flash Attention compared to passing a short input sequence as done in the beginning.
|
||||
|
||||
```py
|
||||
flush()
|
||||
```
|
||||
|
||||
For more information on how to use Flash Attention, please have a look at [this doc page](https://huggingface.co/docs/transformers/en/perf_infer_gpu_one#flashattention-2).
|
||||
|
||||
## 3. Architectural Innovations
|
||||
|
||||
So far we have looked into improving computational and memory efficiency by:
|
||||
|
||||
@ -62,24 +62,13 @@ model.enable_cpu_offload()
|
||||
|
||||
Note that 🤗 Accelerate must be installed before using this feature. [Here's how to install it.](https://huggingface.co/docs/accelerate/basic_tutorials/install)
|
||||
|
||||
#### Using Better Transformer
|
||||
|
||||
Better Transformer is an 🤗 Optimum feature that performs kernel fusion under the hood. You can gain 20% to 30% in speed with zero performance degradation. It only requires one line of code to export the model to 🤗 Better Transformer:
|
||||
|
||||
```python
|
||||
model = model.to_bettertransformer()
|
||||
```
|
||||
|
||||
Note that 🤗 Optimum must be installed before using this feature. [Here's how to install it.](https://huggingface.co/docs/optimum/installation)
|
||||
|
||||
#### Using Flash Attention 2
|
||||
|
||||
Flash Attention 2 is an even faster, optimized version of the previous optimization.
|
||||
|
||||
##### Installation
|
||||
|
||||
First, check whether your hardware is compatible with Flash Attention 2. The latest list of compatible hardware can be found in the [official documentation](https://github.com/Dao-AILab/flash-attention#installation-and-features). If your hardware is not compatible with Flash Attention 2, you can still benefit from attention kernel optimisations through Better Transformer support covered [above](https://huggingface.co/docs/transformers/main/en/model_doc/bark#using-better-transformer).
|
||||
|
||||
First, check whether your hardware is compatible with Flash Attention 2. The latest list of compatible hardware can be found in the [official documentation](https://github.com/Dao-AILab/flash-attention#installation-and-features).
|
||||
Next, [install](https://github.com/Dao-AILab/flash-attention#installation-and-features) the latest version of Flash Attention 2:
|
||||
|
||||
```bash
|
||||
@ -96,7 +85,7 @@ model = BarkModel.from_pretrained("suno/bark-small", dtype=torch.float16, attn_i
|
||||
|
||||
##### Performance comparison
|
||||
|
||||
The following diagram shows the latency for the native attention implementation (no optimisation) against Better Transformer and Flash Attention 2. In all cases, we generate 400 semantic tokens on a 40GB A100 GPU with PyTorch 2.1. Flash Attention 2 is also consistently faster than Better Transformer, and its performance improves even more as batch sizes increase:
|
||||
The following diagram shows the latency for the native attention implementation (no optimisation) against Flash Attention 2. In all cases, we generate 400 semantic tokens on a 40GB A100 GPU with PyTorch 2.1:
|
||||
|
||||
<div style="text-align: center">
|
||||
<img src="https://huggingface.co/datasets/ylacombe/benchmark-comparison/resolve/main/Bark%20Optimization%20Benchmark.png">
|
||||
@ -104,11 +93,9 @@ The following diagram shows the latency for the native attention implementation
|
||||
|
||||
To put this into perspective, on an NVIDIA A100 and when generating 400 semantic tokens with a batch size of 16, you can get 17 times the [throughput](https://huggingface.co/blog/optimizing-bark#throughput) and still be 2 seconds faster than generating sentences one by one with the native model implementation. In other words, all the samples will be generated 17 times faster.
|
||||
|
||||
At batch size 8, on an NVIDIA A100, Flash Attention 2 is also 10% faster than Better Transformer, and at batch size 16, 25%.
|
||||
|
||||
#### Combining optimization techniques
|
||||
|
||||
You can combine optimization techniques, and use CPU offload, half-precision and Flash Attention 2 (or 🤗 Better Transformer) all at once.
|
||||
You can combine optimization techniques, and use CPU offload, half-precision and Flash Attention 2 all at once.
|
||||
|
||||
```python
|
||||
from transformers import BarkModel, infer_device
|
||||
|
||||
@ -73,7 +73,7 @@ Flash Attention 2 is an faster, optimized version of the model.
|
||||
|
||||
### Installation
|
||||
|
||||
First, check whether your hardware is compatible with Flash Attention 2. The latest list of compatible hardware can be found in the [official documentation](https://github.com/Dao-AILab/flash-attention#installation-and-features). If your hardware is not compatible with Flash Attention 2, you can still benefit from attention kernel optimisations through Better Transformer support covered [above](https://huggingface.co/docs/transformers/main/en/model_doc/bark#using-better-transformer).
|
||||
First, check whether your hardware is compatible with Flash Attention 2. The latest list of compatible hardware can be found in the [official documentation](https://github.com/Dao-AILab/flash-attention#installation-and-features).
|
||||
|
||||
Next, [install](https://github.com/Dao-AILab/flash-attention#installation-and-features) the latest version of Flash Attention 2:
|
||||
|
||||
|
||||
@ -55,7 +55,7 @@ Flash Attention 2 is an faster, optimized version of the model.
|
||||
|
||||
### Installation
|
||||
|
||||
First, check whether your hardware is compatible with Flash Attention 2. The latest list of compatible hardware can be found in the [official documentation](https://github.com/Dao-AILab/flash-attention#installation-and-features). If your hardware is not compatible with Flash Attention 2, you can still benefit from attention kernel optimisations through Better Transformer support covered [above](https://huggingface.co/docs/transformers/main/en/model_doc/bark#using-better-transformer).
|
||||
First, check whether your hardware is compatible with Flash Attention 2. The latest list of compatible hardware can be found in the [official documentation](https://github.com/Dao-AILab/flash-attention#installation-and-features).
|
||||
|
||||
Next, [install](https://github.com/Dao-AILab/flash-attention#installation-and-features) the latest version of Flash Attention 2:
|
||||
|
||||
|
||||
@ -39,25 +39,6 @@ pred = onnx_qa(question, context)
|
||||
> [!TIP]
|
||||
> Optimum includes an [Intel](https://hf.co/docs/optimum/intel/index) extension that provides additional optimizations such as quantization, pruning, and knowledge distillation for Intel CPUs. This extension also includes tools to convert models to [OpenVINO](https://hf.co/docs/optimum/intel/inference), a toolkit for optimizing and deploying models, for even faster inference.
|
||||
|
||||
### BetterTransformer
|
||||
|
||||
[BetterTransformer](https://pytorch.org/blog/a-better-transformer-for-fast-transformer-encoder-inference/) is a *fastpath* execution of specialized Transformers functions directly on the hardware level such as a CPU. There are two main components of the fastpath execution.
|
||||
|
||||
- fusing multiple operations into a single kernel for faster and more efficient execution
|
||||
- skipping unnecessary computation of padding tokens with nested tensors
|
||||
|
||||
> [!WARNING]
|
||||
> BetterTransformer isn't supported for all models. Check this [list](https://hf.co/docs/optimum/bettertransformer/overview#supported-models) to see whether a model supports BetterTransformer.
|
||||
|
||||
BetterTransformer is available through Optimum with [`~PreTrainedModel.to_bettertransformer`].
|
||||
|
||||
```py
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
model = AutoModelForCausalLM.from_pretrained("bigscience/bloom")
|
||||
model = model.to_bettertransformer()
|
||||
```
|
||||
|
||||
## TorchScript
|
||||
|
||||
[TorchScript](https://pytorch.org/docs/stable/jit.html) is an intermediate PyTorch model format that can be run in non-Python environments, like C++, where performance is critical. Train a PyTorch model and convert it to a TorchScript function or module with [torch.jit.trace](https://pytorch.org/docs/stable/generated/torch.jit.trace.html). This function optimizes the model with just-in-time (JIT) compilation, and compared to the default eager mode, JIT-compiled models offer better inference performance.
|
||||
|
||||
@ -137,34 +137,6 @@ result = pipeline("Both the music and visual were astounding, not to mention the
|
||||
|
||||
Learn more details about using ORT with Optimum in the [Accelerated inference on NVIDIA GPUs](https://hf.co/docs/optimum/onnxruntime/usage_guides/gpu#accelerated-inference-on-nvidia-gpus) and [Accelerated inference on AMD GPUs](https://hf.co/docs/optimum/onnxruntime/usage_guides/amdgpu#accelerated-inference-on-amd-gpus) guides.
|
||||
|
||||
### BetterTransformer
|
||||
|
||||
[BetterTransformer](https://pytorch.org/blog/a-better-transformer-for-fast-transformer-encoder-inference/) is a *fastpath* execution of specialized Transformers functions directly on the hardware level such as a GPU. There are two main components of the fastpath execution.
|
||||
|
||||
- fusing multiple operations into a single kernel for faster and more efficient execution
|
||||
- skipping unnecessary computation of padding tokens with nested tensors
|
||||
|
||||
> [!WARNING]
|
||||
> Some BetterTransformer features are being upstreamed to Transformers with default support for native [torch.nn.functional.scaled_dot_product_attention](https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html) (SDPA). BetterTransformer has a wider coverage than the Transformers SDPA integration, but you can expect more and more architectures to natively support SDPA in Transformers.
|
||||
|
||||
BetterTransformer is available through Optimum with [`~PreTrainedModel.to_bettertransformer`].
|
||||
|
||||
```py
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
model = AutoModelForCausalLM.from_pretrained("bigscience/bloom")
|
||||
model = model.to_bettertransformer()
|
||||
```
|
||||
|
||||
Call [`~PreTrainedModel.reverse_bettertransformer`] and save it first to return the model to the original Transformers model.
|
||||
|
||||
```py
|
||||
model = model.reverse_bettertransformer()
|
||||
model.save_pretrained("saved_model")
|
||||
```
|
||||
|
||||
Refer to the benchmarks in [Out of the box acceleration and memory savings of 🤗 decoder models with PyTorch 2.0](https://pytorch.org/blog/out-of-the-box-acceleration/) for BetterTransformer and scaled dot product attention performance. The [BetterTransformer](https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2) blog post also discusses fastpath execution in greater detail if you're interested in learning more.
|
||||
|
||||
## Scaled dot product attention (SDPA)
|
||||
|
||||
PyTorch's [torch.nn.functional.scaled_dot_product_attention](https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html) (SDPA) is a native implementation of the scaled dot product attention mechanism. SDPA is a more efficient and optimized version of the attention mechanism used in transformer models.
|
||||
|
||||
@ -17,10 +17,6 @@ rendered properly in your Markdown viewer.
|
||||
|
||||
Questa guida si concentra sull'inferenza di modelli di grandi dimensioni in modo efficiente sulla CPU.
|
||||
|
||||
## `BetterTransformer` per inferenza più rapida
|
||||
|
||||
Abbiamo integrato di recente `BetterTransformer` per fare inferenza più rapidamente con modelli per testi, immagini e audio. Visualizza la documentazione sull'integrazione [qui](https://huggingface.co/docs/optimum/bettertransformer/overview) per maggiori dettagli.
|
||||
|
||||
## PyTorch JIT-mode (TorchScript)
|
||||
|
||||
TorchScript è un modo di creare modelli serializzabili e ottimizzabili da codice PyTorch. Ogni programma TorchScript può esere salvato da un processo Python e caricato in un processo dove non ci sono dipendenze Python.
|
||||
|
||||
@ -22,7 +22,3 @@ Questo documento contiene informazioni su come fare inferenza in maniera efficie
|
||||
Nota: Un setup con GPU multiple può utilizzare la maggior parte delle strategie descritte nella [sezione con GPU singola](./perf_infer_gpu_one). Tuttavia, è necessario conoscere delle tecniche semplici che possono essere utilizzate per un risultato migliore.
|
||||
|
||||
</Tip>
|
||||
|
||||
## `BetterTransformer` per inferenza più rapida
|
||||
|
||||
Abbiamo recentemente integrato `BetterTransformer` per inferenza più rapida su multi-GPU per modelli su testo, immagini e audio. Controlla il documento con queste integrazioni [qui](https://huggingface.co/docs/optimum/bettertransformer/overview) per maggiori dettagli.
|
||||
|
||||
@ -17,10 +17,6 @@ rendered properly in your Markdown viewer.
|
||||
|
||||
Questo documento sarà presto completato con informazioni su come effetture l'inferenza su una singola GPU. Nel frattempo è possibile consultare [la guida per l'addestramento su una singola GPU](perf_train_gpu_one) e [la guida per l'inferenza su CPU](perf_infer_cpu).
|
||||
|
||||
## `BetterTransformer` per l'inferenza più veloce
|
||||
|
||||
Abbiamo recentemente integrato `BetterTransformer` per velocizzare l'inferenza su GPU per modelli di testo, immagini e audio. Per maggiori dettagli, consultare la documentazione su questa integrazione [qui](https://huggingface.co/docs/optimum/bettertransformer/overview).
|
||||
|
||||
## Integrazione di `bitsandbytes` per Int8 mixed-precision matrix decomposition
|
||||
|
||||
<Tip>
|
||||
|
||||
@ -42,16 +42,6 @@ device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
model = BarkModel.from_pretrained("suno/bark-small", dtype=torch.float16).to(device)
|
||||
```
|
||||
|
||||
#### Using 🤗 Better Transformer
|
||||
|
||||
Better Transformer は、内部でカーネル融合を実行する 🤗 最適な機能です。パフォーマンスを低下させることなく、速度を 20% ～ 30% 向上させることができます。モデルを 🤗 Better Transformer にエクスポートするのに必要なコードは 1 行だけです。
|
||||
|
||||
```python
|
||||
model = model.to_bettertransformer()
|
||||
```
|
||||
|
||||
この機能を使用する前に 🤗 Optimum をインストールする必要があることに注意してください。 [インストール方法はこちら](https://huggingface.co/docs/optimum/installation)
|
||||
|
||||
#### Using CPU offload
|
||||
|
||||
前述したように、Bark は 4 つのサブモデルで構成されており、オーディオ生成中に順番に呼び出されます。言い換えれば、1 つのサブモデルが使用されている間、他のサブモデルはアイドル状態になります。
|
||||
@ -64,27 +54,6 @@ model.enable_cpu_offload()
|
||||
|
||||
この機能を使用する前に、🤗 Accelerate をインストールする必要があることに注意してください。 [インストール方法はこちら](https://huggingface.co/docs/accelerate/basic_tutorials/install)
|
||||
|
||||
#### Combining optimization techniques
|
||||
|
||||
最適化手法を組み合わせて、CPU オフロード、半精度、🤗 Better Transformer をすべて一度に使用できます。
|
||||
|
||||
```python
|
||||
from transformers import BarkModel
|
||||
import torch
|
||||
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
# load in fp16
|
||||
model = BarkModel.from_pretrained("suno/bark-small", dtype=torch.float16).to(device)
|
||||
|
||||
# convert to bettertransformer
|
||||
model = BetterTransformer.transform(model, keep_original_model=False)
|
||||
|
||||
# enable CPU offload
|
||||
model.enable_cpu_offload()
|
||||
```
|
||||
|
||||
推論最適化手法の詳細については、[こちら](https://huggingface.co/docs/transformers/perf_infer_gpu_one) をご覧ください。
|
||||
|
||||
### Tips
|
||||
|
||||
|
||||
@ -18,10 +18,6 @@ rendered properly in your Markdown viewer.
|
||||
|
||||
このガイドは、CPU上で大規模なモデルの効率的な推論に焦点を当てています。
|
||||
|
||||
## `BetterTransformer` for faster inference
|
||||
|
||||
最近、テキスト、画像、および音声モデルのCPU上での高速な推論のために`BetterTransformer`を統合しました。詳細については、この統合に関するドキュメンテーションを[こちら](https://huggingface.co/docs/optimum/bettertransformer/overview)で確認してください。
|
||||
|
||||
## PyTorch JITモード（TorchScript）
|
||||
TorchScriptは、PyTorchコードからシリアライズ可能で最適化可能なモデルを作成する方法です。任意のTorchScriptプログラムは、Python依存性のないプロセスで保存およびロードできます。
|
||||
デフォルトのイーガーモードと比較して、PyTorchのjitモードは通常、オペレーターフュージョンなどの最適化手法によりモデル推論のパフォーマンスが向上します。
|
||||
|
||||
@ -25,103 +25,3 @@ rendered properly in your Markdown viewer.
|
||||
## Flash Attention 2
|
||||
|
||||
Flash Attention 2の統合は、複数のGPUセットアップでも機能します。詳細については、[単一のGPUセクション](./perf_infer_gpu_one#Flash-Attention-2)の適切なセクションをご覧ください。
|
||||
|
||||
## BetterTransformer
|
||||
|
||||
[BetterTransformer](https://huggingface.co/docs/optimum/bettertransformer/overview)は、🤗 TransformersモデルをPyTorchネイティブの高速実行パスを使用するように変換し、その下でFlash Attentionなどの最適化されたカーネルを呼び出します。
|
||||
|
||||
BetterTransformerは、テキスト、画像、音声モデルの単一GPUおよび複数GPUでの高速推論もサポートしています。
|
||||
<Tip>
|
||||
|
||||
Flash Attentionは、fp16またはbf16 dtypeを使用しているモデルにのみ使用できます。BetterTransformerを使用する前に、モデルを適切なdtypeにキャストしてください。
|
||||
|
||||
</Tip>
|
||||
|
||||
### Decoder models
|
||||
|
||||
テキストモデル、特にデコーダーベースのモデル（GPT、T5、Llamaなど）の場合、BetterTransformer APIはすべての注意操作を[`torch.nn.functional.scaled_dot_product_attention`オペレーター](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention)（SDPA）を使用するように変換します。これはPyTorch 2.0以降でのみ使用可能です。
|
||||
|
||||
モデルをBetterTransformerに変換するには：
|
||||
|
||||
```python
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
model = AutoModelForCausalLM.from_pretrained("facebook/opt-350m")
|
||||
# convert the model to BetterTransformer
|
||||
model.to_bettertransformer()
|
||||
|
||||
# Use it for training or inference
|
||||
```
|
||||
|
||||
SDPAは、ハードウェアや問題のサイズなどの特定の設定で[Flash Attention](https://huggingface.co/papers/2205.14135)カーネルを呼び出すこともできます。Flash Attentionを有効にするか、特定の設定（ハードウェア、問題のサイズ）で利用可能かを確認するには、[`torch.nn.kernel.sdpa_kernel`](https://pytorch.org/docs/stable/generated/torch.nn.attention.sdpa_kernel.html)をコンテキストマネージャとして使用します。
|
||||
|
||||
|
||||
```diff
|
||||
import torch
|
||||
+ from torch.nn.attention import SDPBackend, sdpa_kernel
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("facebook/opt-350m")
|
||||
model = AutoModelForCausalLM.from_pretrained("facebook/opt-350m").to("cuda")
|
||||
# convert the model to BetterTransformer
|
||||
model.to_bettertransformer()
|
||||
|
||||
input_text = "Hello my dog is cute and"
|
||||
inputs = tokenizer(input_text, return_tensors="pt").to("cuda")
|
||||
|
||||
+ with sdpa_kernel(SDPBackend.FLASH_ATTENTION):
|
||||
outputs = model.generate(**inputs)
|
||||
|
||||
print(tokenizer.decode(outputs[0], skip_special_tokens=True))
|
||||
```
|
||||
|
||||
もしトレースバックで次のようなエラーメッセージが表示された場合：
|
||||
|
||||
|
||||
```bash
|
||||
RuntimeError: No available kernel. Aborting execution.
|
||||
```
|
||||
|
||||
当日、Flash Attentionのカバレッジが広範囲である可能性があるPyTorch Nightlyバージョンを試すようにお勧めします。
|
||||
|
||||
```bash
|
||||
pip3 install -U --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118
|
||||
```
|
||||
|
||||
[このブログ投稿](https://pytorch.org/blog/out-of-the-box-acceleration/)をチェックして、BetterTransformer + SDPA APIで可能なことについて詳しく学びましょう。
|
||||
|
||||
### Encoder Models
|
||||
|
||||
推論中のエンコーダーモデルでは、BetterTransformerはエンコーダーレイヤーのforward呼び出しを、エンコーダーレイヤーの[`torch.nn.TransformerEncoderLayer`](https://pytorch.org/docs/stable/generated/torch.nn.TransformerEncoderLayer.html)の相当するものにディスパッチします。これにより、エンコーダーレイヤーの高速実装が実行されます。
|
||||
|
||||
`torch.nn.TransformerEncoderLayer`の高速実装はトレーニングをサポートしていないため、代わりに`torch.nn.functional.scaled_dot_product_attention`にディスパッチされます。これにより、ネストされたテンソルを活用しないFlash AttentionまたはMemory-Efficient Attentionの融合カーネルを使用できます。
|
||||
|
||||
BetterTransformerのパフォーマンスの詳細については、この[ブログ投稿](https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2)をご覧いただけます。また、エンコーダーモデル用のBetterTransformerについては、この[ブログ](https://pytorch.org/blog/a-better-transformer-for-fast-transformer-encoder-inference/)で詳しく学ぶことができます。
|
||||
|
||||
|
||||
## Advanced usage: mixing FP4 (or Int8) and BetterTransformer
|
||||
|
||||
モデルの最良のパフォーマンスを得るために、上記で説明した異なる方法を組み合わせることができます。例えば、FP4ミックスプレシジョン推論+Flash Attentionを使用したBetterTransformerを組み合わせることができます。
|
||||
|
||||
|
||||
```py
|
||||
import torch
|
||||
from torch.nn.attention import SDPBackend, sdpa_kernel
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
|
||||
|
||||
quantization_config = BitsAndBytesConfig(
|
||||
load_in_4bit=True,
|
||||
bnb_4bit_compute_dtype=torch.float16
|
||||
)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("facebook/opt-350m")
|
||||
model = AutoModelForCausalLM.from_pretrained("facebook/opt-350m", quantization_config=quantization_config)
|
||||
|
||||
input_text = "Hello my dog is cute and"
|
||||
inputs = tokenizer(input_text, return_tensors="pt").to("cuda")
|
||||
|
||||
with sdpa_kernel(SDPBackend.FLASH_ATTENTION):
|
||||
outputs = model.generate(**inputs)
|
||||
|
||||
print(tokenizer.decode(outputs[0], skip_special_tokens=True))
|
||||
```
|
||||
@ -19,12 +19,6 @@ rendered properly in your Markdown viewer.
|
||||
|
||||
## Flash Attention 2
|
||||
|
||||
<Tip>
|
||||
|
||||
この機能は実験的であり、将来のバージョンで大幅に変更される可能性があります。たとえば、Flash Attention 2 APIは近い将来`BetterTransformer` APIに移行するかもしれません。
|
||||
|
||||
</Tip>
|
||||
|
||||
Flash Attention 2は、トランスフォーマーベースのモデルのトレーニングと推論速度を大幅に高速化できます。Flash Attention 2は、Tri Dao氏によって[公式のFlash Attentionリポジトリ](https://github.com/Dao-AILab/flash-attention)で導入されました。Flash Attentionに関する科学論文は[こちら](https://huggingface.co/papers/2205.14135)で見ることができます。
|
||||
|
||||
Flash Attention 2を正しくインストールするには、上記のリポジトリに記載されているインストールガイドに従ってください。
|
||||
@ -34,8 +28,6 @@ Flash Attention 2を正しくインストールするには、上記のリポジ
|
||||
- Llama
|
||||
- Falcon
|
||||
|
||||
さらに多くのモデルにFlash Attention 2のサポートを追加することをGitHubで提案することもでき、変更を統合するためにプルリクエストを開くこともできます。サポートされているモデルは、パディングトークンを使用してトレーニングを含む、推論とトレーニングに使用できます（現在の`BetterTransformer` APIではサポートされていない）。
|
||||
|
||||
<Tip>
|
||||
|
||||
Flash Attention 2は、モデルのdtypeが`fp16`または`bf16`の場合にのみ使用でき、NVIDIA-GPUデバイスでのみ実行されます。この機能を使用する前に、モデルを適切なdtypeにキャストし、サポートされているデバイスにロードしてください。
|
||||
@ -164,98 +156,6 @@ model.add_adapter(lora_config)
|
||||
... # train your model
|
||||
```
|
||||
|
||||
## BetterTransformer
|
||||
|
||||
[BetterTransformer](https://huggingface.co/docs/optimum/bettertransformer/overview)は、🤗 TransformersモデルをPyTorchネイティブの高速パス実行に変換します。これにより、Flash Attentionなどの最適化されたカーネルが内部で呼び出されます。
|
||||
|
||||
BetterTransformerは、テキスト、画像、およびオーディオモデルの単一およびマルチGPUでの高速な推論をサポートしています。
|
||||
|
||||
<Tip>
|
||||
|
||||
Flash Attentionは、fp16またはbf16のdtypeを使用するモデルにのみ使用できます。BetterTransformerを使用する前に、モデルを適切なdtypeにキャストしてください。
|
||||
|
||||
</Tip>
|
||||
|
||||
### Encoder models
|
||||
|
||||
PyTorchネイティブの[`nn.MultiHeadAttention`](https://pytorch.org/blog/a-better-transformer-for-fast-transformer-encoder-inference/)アテンション高速パス、BetterTransformerと呼ばれるものは、[🤗 Optimumライブラリ](https://huggingface.co/docs/optimum/bettertransformer/overview)の統合を通じてTransformersと一緒に使用できます。
|
||||
|
||||
PyTorchのアテンション高速パスを使用すると、カーネルフュージョンと[ネストされたテンソル](https://pytorch.org/docs/stable/nested.html)の使用により、推論を高速化できます。詳細なベンチマーク情報は[このブログ記事](https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2)にあります。
|
||||
|
||||
[`optimum`](https://github.com/huggingface/optimum)パッケージをインストールした後、推論中にBetter Transformerを使用するには、関連する内部モジュールを呼び出すことで置き換える必要があります[`~PreTrainedModel.to_bettertransformer`]:
|
||||
|
||||
|
||||
```python
|
||||
model = model.to_bettertransformer()
|
||||
```
|
||||
|
||||
メソッド [`~PreTrainedModel.reverse_bettertransformer`] は、モデルを保存する前に使用すべきで、標準のトランスフォーマーモデリングを使用するためのものです：
|
||||
|
||||
```python
|
||||
model = model.reverse_bettertransformer()
|
||||
model.save_pretrained("saved_model")
|
||||
```
|
||||
|
||||
BetterTransformer APIを使ったエンコーダーモデルの可能性について詳しく知るには、[このブログポスト](https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2)をご覧ください。
|
||||
|
||||
### Decoder models
|
||||
|
||||
テキストモデル、特にデコーダーベースのモデル（GPT、T5、Llamaなど）にとって、BetterTransformer APIはすべての注意操作を[`torch.nn.functional.scaled_dot_product_attention`オペレーター](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention)（SDPA）を使用するように変換します。このオペレーターはPyTorch 2.0以降でのみ利用可能です。
|
||||
|
||||
モデルをBetterTransformerに変換するには、以下の手順を実行してください：
|
||||
|
||||
```python
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
model = AutoModelForCausalLM.from_pretrained("facebook/opt-350m")
|
||||
# convert the model to BetterTransformer
|
||||
model.to_bettertransformer()
|
||||
|
||||
# Use it for training or inference
|
||||
```
|
||||
|
||||
SDPAは、ハードウェアや問題のサイズに応じて[Flash Attention](https://huggingface.co/papers/2205.14135)カーネルを使用することもできます。Flash Attentionを有効にするか、特定の設定（ハードウェア、問題サイズ）で使用可能かどうかを確認するには、[`torch.nn.attention.sdpa_kernel`](https://pytorch.org/docs/stable/generated/torch.nn.attention.sdpa_kernel.html)をコンテキストマネージャとして使用します。
|
||||
|
||||
|
||||
```diff
|
||||
import torch
|
||||
+ from torch.nn.attention import SDPBackend, sdpa_kernel
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("facebook/opt-350m")
|
||||
model = AutoModelForCausalLM.from_pretrained("facebook/opt-350m", dtype=torch.float16).to("cuda")
|
||||
# convert the model to BetterTransformer
|
||||
model.to_bettertransformer()
|
||||
|
||||
input_text = "Hello my dog is cute and"
|
||||
inputs = tokenizer(input_text, return_tensors="pt").to("cuda")
|
||||
|
||||
+ with sdpa_kernel(SDPBackend.FLASH_ATTENTION):
|
||||
outputs = model.generate(**inputs)
|
||||
|
||||
print(tokenizer.decode(outputs[0], skip_special_tokens=True))
|
||||
```
|
||||
|
||||
もしトレースバックにバグが表示された場合
|
||||
|
||||
```bash
|
||||
RuntimeError: No available kernel. Aborting execution.
|
||||
```
|
||||
|
||||
Flash Attention の広範なカバレッジを持つかもしれない PyTorch のナイトリーバージョンを試してみることをお勧めします。
|
||||
|
||||
```bash
|
||||
pip3 install -U --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118
|
||||
```
|
||||
|
||||
Or make sure your model is correctly casted in float16 or bfloat16
|
||||
|
||||
モデルが正しくfloat16またはbfloat16にキャストされていることを確認してください。
|
||||
|
||||
Have a look at [this detailed blogpost](https://pytorch.org/blog/out-of-the-box-acceleration/) to read more about what is possible to do with `BetterTransformer` + SDPA API.
|
||||
|
||||
`BetterTransformer` + SDPA APIを使用して何が可能かについて詳しく読むには、[この詳細なブログポスト](https://pytorch.org/blog/out-of-the-box-acceleration/)をご覧ください。
|
||||
|
||||
## `bitsandbytes` integration for FP4 mixed-precision inference
|
||||
|
||||
FP4混合精度推論のための`bitsandbytes`統合
|
||||
@ -415,29 +315,3 @@ In this example, the first GPU will use 1GB of memory and the second 2GB.
|
||||
|
||||
[](https://colab.research.google.com/drive/1qOjXfQIAULfKvZqwCen8-MoWKGdSatZ4?usp=sharing)
|
||||
|
||||
## Advanced usage: mixing FP4 (or Int8) and BetterTransformer
|
||||
|
||||
異なる方法を組み合わせて、モデルの最適なパフォーマンスを得ることができます。例えば、BetterTransformerを使用してFP4ミックスプレシジョン推論とフラッシュアテンションを組み合わせることができます。
|
||||
|
||||
|
||||
```py
|
||||
import torch
|
||||
from torch.nn.attention import SDPBackend, sdpa_kernel
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
|
||||
|
||||
quantization_config = BitsAndBytesConfig(
|
||||
load_in_4bit=True,
|
||||
bnb_4bit_compute_dtype=torch.float16
|
||||
)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("facebook/opt-350m")
|
||||
model = AutoModelForCausalLM.from_pretrained("facebook/opt-350m", quantization_config=quantization_config)
|
||||
|
||||
input_text = "Hello my dog is cute and"
|
||||
inputs = tokenizer(input_text, return_tensors="pt").to("cuda")
|
||||
|
||||
with sdpa_kernel(SDPBackend.FLASH_ATTENTION):
|
||||
outputs = model.generate(**inputs)
|
||||
|
||||
print(tokenizer.decode(outputs[0], skip_special_tokens=True))
|
||||
```
|
||||
@ -57,7 +57,6 @@ rendered properly in your Markdown viewer.
|
||||
これらの方法が十分な利益をもたらさない場合、以下のオプションを検討できます：
|
||||
* [効率的なソフトウェアプリビルドを備えたカスタムDockerコンテナの作成](#efficient-software-prebuilds)
|
||||
* [Mixture of Experts（MoE）を使用するモデルを検討](#mixture-of-experts)
|
||||
* [モデルをBetterTransformerに変換して、PyTorchネイティブのアテンションを活用](#using-pytorch-native-attention)
|
||||
|
||||
最後に、これらの方法がまだ十分でない場合、A100などのサーバーグレードGPUに切り替えても、さらなる改善が必要かもしれません。これらのアプローチは、マルチGPUセットアップでも有効であり、[マルチGPUセクション](perf_train_gpu_many)で説明されている追加の並列化技術を活用できます。
|
||||
|
||||
@ -412,26 +411,3 @@ PyTorchの[pipとcondaビルド](https://pytorch.org/get-started/locally/#start-
|
||||
|
||||
PytorchにはDeepSpeedが構築したものもあります: [DeepSpeed-MoE: Advancing Mixture-of-Experts Inference and Training to Power Next-Generation AI Scale](https://huggingface.co/papers/2201.05596)、[Mixture of Experts](https://www.deepspeed.ai/tutorials/mixture-of-experts/) - ブログ記事: [1](https://www.microsoft.com/en-us/research/blog/deepspeed-powers-8x-larger-moe-model-training-with-high-performance/)、[2](https://www.microsoft.com/en-us/research/publication/scalable-and-efficient-moe-training-for-multitask-multilingual-models/)、大規模なTransformerベースの自然言語生成モデルの具体的な展開については、[ブログ記事](https://www.deepspeed.ai/2021/12/09/deepspeed-moe-nlg.html)、[Megatron-Deepspeedブランチ](https://github.com/microsoft/Megatron-DeepSpeed/tree/moe-training)を参照してください。
|
||||
|
||||
|
||||
## PyTorchネイティブアテンションとFlash Attentionの使用
|
||||
|
||||
PyTorch 2.0では、ネイティブの[`torch.nn.functional.scaled_dot_product_attention`](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention.html)（SDPA）がリリースされ、[メモリ効率の高いアテンション](https://huggingface.co/papers/2112.05682)や[フラッシュアテンション](https://huggingface.co/papers/2205.14135)などの融合されたGPUカーネルの使用を可能にします。
|
||||
|
||||
[`optimum`](https://github.com/huggingface/optimum)パッケージをインストールした後、関連する内部モジュールを置き換えて、PyTorchのネイティブアテンションを使用できます。以下のように設定します：
|
||||
|
||||
|
||||
```python
|
||||
model = model.to_bettertransformer()
|
||||
```
|
||||
|
||||
変換後、通常通りモデルをトレーニングしてください。
|
||||
|
||||
<Tip warning={true}>
|
||||
|
||||
PyTorchネイティブの`scaled_dot_product_attention`演算子は、`attention_mask`が提供されていない場合にのみFlash Attentionにディスパッチできます。
|
||||
|
||||
デフォルトでは、トレーニングモードでBetterTransformer統合はマスクサポートを削除し、バッチトレーニングにパディングマスクが必要ないトレーニングにしか使用できません。これは、例えばマスク言語モデリングや因果言語モデリングのような、バッチトレーニングにパディングマスクが不要なトレーニングの場合に該当します。BetterTransformerはパディングマスクが必要なタスクに対するモデルの微調整には適していません。
|
||||
|
||||
</Tip>
|
||||
|
||||
SDPAを使用したアクセラレーションとメモリの節約について詳しく知りたい場合は、この[ブログ記事](https://pytorch.org/blog/out-of-the-box-acceleration/)をチェックしてください。
|
||||
|
||||
@ -320,160 +320,6 @@ $$ \textbf{O}_i \leftarrow s^a_{ij} * \textbf{O}_i + s^b_{ij} * \mathbf{V}_{j} \
|
||||
|
||||
실제 예를 살펴보겠습니다.
|
||||
|
||||
우리의 OctoCoder 모델은 이제 *시스템 프롬프트*가 포함된 훨씬 더 긴 입력 프롬프트를 받게 됩니다. 시스템 프롬프트는 대규모 언어 모델을 사용자의 작업에 맞춘 더 나은 어시스턴트로 유도하는 데 사용됩니다. 다음 예제에서는 OctoCoder를 더 나은 코딩 어시스턴트로 만들기 위한 시스템 프롬프트를 사용합니다.
|
||||
|
||||
```python
|
||||
system_prompt = """Below are a series of dialogues between various people and an AI technical assistant.
|
||||
The assistant tries to be helpful, polite, honest, sophisticated, emotionally aware, and humble but knowledgeable.
|
||||
The assistant is happy to help with code questions and will do their best to understand exactly what is needed.
|
||||
It also tries to avoid giving false or misleading information, and it caveats when it isn't entirely sure about the right answer.
|
||||
That said, the assistant is practical really does its best, and doesn't let caution get too much in the way of being useful.
|
||||
|
||||
The Starcoder models are a series of 15.5B parameter models trained on 80+ programming languages from The Stack (v1.2) (excluding opt-out requests).
|
||||
The model uses Multi Query Attention, was trained using the Fill-in-the-Middle objective, and with 8,192 tokens context window for a trillion tokens of heavily deduplicated data.
|
||||
|
||||
-----
|
||||
|
||||
Question: Write a function that takes two lists and returns a list that has alternating elements from each input list.
|
||||
|
||||
Answer: Sure. Here is a function that does that.
|
||||
|
||||
def alternating(list1, list2):
|
||||
results = []
|
||||
for i in range(len(list1)):
|
||||
results.append(list1[i])
|
||||
results.append(list2[i])
|
||||
return results
|
||||
|
||||
Question: Can you write some test cases for this function?
|
||||
|
||||
Answer: Sure, here are some tests.
|
||||
|
||||
assert alternating([10, 20, 30], [1, 2, 3]) == [10, 1, 20, 2, 30, 3]
|
||||
assert alternating([True, False], [4, 5]) == [True, 4, False, 5]
|
||||
assert alternating([], []) == []
|
||||
|
||||
Question: Modify the function so that it returns all input elements when the lists have uneven length. The elements from the longer list should be at the end.
|
||||
|
||||
Answer: Here is the modified function.
|
||||
|
||||
def alternating(list1, list2):
|
||||
results = []
|
||||
for i in range(min(len(list1), len(list2))):
|
||||
results.append(list1[i])
|
||||
results.append(list2[i])
|
||||
if len(list1) > len(list2):
|
||||
results.extend(list1[i+1:])
|
||||
else:
|
||||
results.extend(list2[i+1:])
|
||||
return results
|
||||
|
||||
-----
|
||||
"""
|
||||
```
|
||||
시연을 위해 시스템 프롬프트를 10번 중복하여 증가시켜 플래시 어텐션의 메모리 절약 효과를 관찰할 수 있을 만큼 입력 길이를 충분히 길게 만듭니다. 원래의 텍스트 프롬프트를 다음과 같이 추가합니다. `"Question: Please write a function in Python that transforms bytes to Giga bytes.\n\nAnswer: Here"`
|
||||
|
||||
```python
|
||||
long_prompt = 10 * system_prompt + prompt
|
||||
```
|
||||
|
||||
모델을 다시 bfloat16 정밀도로 인스턴스화합니다.
|
||||
|
||||
```python
|
||||
model = AutoModelForCausalLM.from_pretrained("bigcode/octocoder", dtype=torch.bfloat16, device_map="auto")
|
||||
tokenizer = AutoTokenizer.from_pretrained("bigcode/octocoder")
|
||||
|
||||
pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
|
||||
```
|
||||
|
||||
이제 플래시 어텐션을 *사용하지 않고* 이전과 동일하게 모델을 실행하여 최대 GPU 메모리 요구량과 추론 시간을 측정해 봅시다.
|
||||
|
||||
```python
|
||||
import time
|
||||
|
||||
start_time = time.time()
|
||||
result = pipe(long_prompt, max_new_tokens=60)[0]["generated_text"][len(long_prompt):]
|
||||
|
||||
print(f"Generated in {time.time() - start_time} seconds.")
|
||||
result
|
||||
```
|
||||
|
||||
**출력**:
|
||||
```
|
||||
Generated in 10.96854019165039 seconds.
|
||||
Sure. Here is a function that does that.\n\ndef bytes_to_giga(bytes):\n return bytes / 1024 / 1024 / 1024\n\nAnswer: Sure. Here is a function that does that.\n\ndef
|
||||
````
|
||||
|
||||
이전과 동일한 출력을 얻고 있지만, 이번에는 모델이 답변을 여러 번 반복하여 60개의 토큰이 잘릴 때까지 계속됩니다. 시연을 위해 시스템 프롬프트를 10번 반복했기 때문에 모델이 스스로 반복하도록 유도한 결과입니다. 이는 놀라운 일이 아닙니다.
|
||||
|
||||
**참고** 실제 응용에서는 시스템 프롬프트를 10번 반복할 필요가 없습니다. 한 번만 사용하면 충분합니다!
|
||||
|
||||
최대 GPU 메모리 요구량을 측정해 봅시다.
|
||||
|
||||
```python
|
||||
bytes_to_giga_bytes(torch.cuda.max_memory_allocated())
|
||||
```
|
||||
|
||||
**출력**:
|
||||
```bash
|
||||
37.668193340301514
|
||||
```
|
||||
|
||||
보시다시피 최대 GPU 메모리 요구량이 처음보다 상당히 높아졌습니다. 이는 주로 입력 시퀀스가 길어졌기 때문입니다. 또한 생성 시간이 이제 1분을 넘어갑니다.
|
||||
|
||||
다음 실험을 위해 `flush()`를 호출하여 GPU 메모리를 초기화합니다.
|
||||
|
||||
```python
|
||||
flush()
|
||||
```
|
||||
|
||||
비교를 위해, 동일한 기능을 실행하되 플래시 어텐션을 활성화해 보겠습니다.
|
||||
이를 위해 모델을 [BetterTransformer](https://huggingface.co/docs/optimum/bettertransformer/overview)로 변환하고, 이를 통해 PyTorch의 [SDPA self-attention](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention)을 활성화하면 플래시 어텐션을 사용할 수 있습니다.
|
||||
|
||||
```python
|
||||
model.to_bettertransformer()
|
||||
```
|
||||
|
||||
이제 이전과 동일한 코드 스니펫을 실행하면, 내부적으로 Transformers가 플래시 어텐션을 사용할 것입니다.
|
||||
|
||||
```py
|
||||
start_time = time.time()
|
||||
with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):
|
||||
result = pipe(long_prompt, max_new_tokens=60)[0]["generated_text"][len(long_prompt):]
|
||||
|
||||
print(f"Generated in {time.time() - start_time} seconds.")
|
||||
result
|
||||
```
|
||||
|
||||
**출력**:
|
||||
```
|
||||
Generated in 3.0211617946624756 seconds.
|
||||
Sure. Here is a function that does that.\n\ndef bytes_to_giga(bytes):\n return bytes / 1024 / 1024 / 1024\n\nAnswer: Sure. Here is a function that does that.\n\ndef
|
||||
```
|
||||
|
||||
이전과 동일한 결과를 얻었지만, 플래시 어텐션 덕분에 매우 큰 속도 향상을 관찰할 수 있습니다.
|
||||
|
||||
메모리 소비량을 마지막으로 한 번 더 측정해 봅시다.
|
||||
|
||||
```python
|
||||
bytes_to_giga_bytes(torch.cuda.max_memory_allocated())
|
||||
```
|
||||
|
||||
**출력**:
|
||||
```
|
||||
32.617331981658936
|
||||
```
|
||||
|
||||
그리고 우리는 처음에 보았던 GPU 메모리 요구량인 29GB로 돌아왔습니다.
|
||||
|
||||
플래시 어텐션을 사용하여 매우 긴 입력 시퀀스를 전달할 때 처음에 짧은 입력 시퀀스를 전달했을 때와 비교하여 약 100MB 정도의 GPU 메모리를 더 사용한다는 것을 관찰할 수 있습니다.
|
||||
|
||||
```py
|
||||
flush()
|
||||
```
|
||||
|
||||
플래시 어텐션 사용에 대한 자세한 정보는 [이 문서 페이지](https://huggingface.co/docs/transformers/en/perf_infer_gpu_one#flashattention-2)를 참조해 주세요.
|
||||
|
||||
## 3. 아키텍처 혁신 [[3-architectural-innovations]]
|
||||
|
||||
지금까지 우리는 계산 및 메모리 효율성을 개선하기 위해 다음을 살펴보았습니다:
|
||||
|
||||
@ -17,10 +17,6 @@ rendered properly in your Markdown viewer.
|
||||
|
||||
이 가이드는 CPU에서 대규모 모델을 효율적으로 추론하는 방법에 중점을 두고 있습니다.
|
||||
|
||||
## 더 빠른 추론을 위한 `BetterTransformer` [[bettertransformer-for-faster-inference]]
|
||||
|
||||
우리는 최근 CPU에서 텍스트, 이미지 및 오디오 모델의 빠른 추론을 위해 `BetterTransformer`를 통합했습니다. 이 통합에 대한 더 자세한 내용은 [이 문서](https://huggingface.co/docs/optimum/bettertransformer/overview)를 참조하세요.
|
||||
|
||||
## PyTorch JIT 모드 (TorchScript) [[pytorch-jitmode-torchscript]]
|
||||
TorchScript는 PyTorch 코드에서 직렬화와 최적화가 가능한 모델을 생성할때 쓰입니다. TorchScript로 만들어진 프로그램은 기존 Python 프로세스에서 저장한 뒤, 종속성이 없는 새로운 프로세스로 가져올 수 있습니다. PyTorch의 기본 설정인 `eager` 모드와 비교했을때, `jit` 모드는 연산자 결합과 같은 최적화 방법론을 통해 모델 추론에서 대부분 더 나은 성능을 제공합니다.
|
||||
|
||||
|
||||
@ -17,27 +17,6 @@ rendered properly in your Markdown viewer.
|
||||
|
||||
이 가이드 외에도, [단일 GPU에서의 훈련 가이드](perf_train_gpu_one)와 [CPU에서의 추론 가이드](perf_infer_cpu)에서도 관련 정보를 찾을 수 있습니다.
|
||||
|
||||
## Better Transformer: PyTorch 네이티브 Transformer 패스트패스 [[better-transformer-pytorchnative-transformer-fastpath]]
|
||||
|
||||
PyTorch 네이티브 [`nn.MultiHeadAttention`](https://pytorch.org/blog/a-better-transformer-for-fast-transformer-encoder-inference/) 어텐션 패스트패스인 BetterTransformer는 [🤗 Optimum 라이브러리](https://huggingface.co/docs/optimum/bettertransformer/overview)의 통합을 통해 Transformers와 함께 사용할 수 있습니다.
|
||||
|
||||
PyTorch의 어텐션 패스트패스는 커널 퓨전과 [중첩된 텐서](https://pytorch.org/docs/stable/nested.html)의 사용을 통해 추론 속도를 높일 수 있습니다. 자세한 벤치마크는 [이 블로그 글](https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2)에서 확인할 수 있습니다.
|
||||
|
||||
[`optimum`](https://github.com/huggingface/optimum) 패키지를 설치한 후에는 추론 중 Better Transformer를 사용할 수 있도록 [`~PreTrainedModel.to_bettertransformer`]를 호출하여 관련 내부 모듈을 대체합니다:
|
||||
|
||||
```python
|
||||
model = model.to_bettertransformer()
|
||||
```
|
||||
|
||||
[`~PreTrainedModel.reverse_bettertransformer`] 메소드는 정규화된 transformers 모델링을 사용하기 위해 모델을 저장하기 전 원래의 모델링으로 돌아갈 수 있도록 해줍니다:
|
||||
|
||||
```python
|
||||
model = model.reverse_bettertransformer()
|
||||
model.save_pretrained("saved_model")
|
||||
```
|
||||
|
||||
PyTorch 2.0부터는 어텐션 패스트패스가 인코더와 디코더 모두에서 지원됩니다. 지원되는 아키텍처 목록은 [여기](https://huggingface.co/docs/optimum/bettertransformer/overview#supported-models)에서 확인할 수 있습니다.
|
||||
|
||||
## FP4 혼합 정밀도 추론을 위한 `bitsandbytes` 통합 [[bitsandbytes-integration-for-fp4-mixedprecision-inference]]
|
||||
|
||||
`bitsandbytes`를 설치하면 GPU에서 손쉽게 모델을 압축할 수 있습니다. FP4 양자화를 사용하면 원래의 전체 정밀도 버전과 비교하여 모델 크기를 최대 8배 줄일 수 있습니다. 아래에서 시작하는 방법을 확인하세요.
|
||||
|
||||
@ -97,7 +97,6 @@ from .utils import (
|
||||
is_flash_attn_3_available,
|
||||
is_kernels_available,
|
||||
is_offline_mode,
|
||||
is_optimum_available,
|
||||
is_peft_available,
|
||||
is_remote_url,
|
||||
is_torch_flex_attn_available,
|
||||
@ -2269,9 +2268,9 @@ class PreTrainedModel(nn.Module, EmbeddingAccessMixin, ModuleUtilsMixin, PushToH
|
||||
Args:
|
||||
is_init_check (`bool`, *optional*):
|
||||
Whether this check is performed early, i.e. at __init__ time, or later when the model and its weights are
|
||||
fully instantiated. This is needed as we also check the devices of the weights, and/or if the model uses
|
||||
BetterTransformer, which are only available later after __init__. This allows to raise proper exceptions early
|
||||
before instantiating the full models if we know that the model does not support the requested attention.
|
||||
fully instantiated. This is needed as we also check the devices of the weights, which are only available
|
||||
later after __init__. This allows to raise proper exceptions early before instantiating the full models
|
||||
if we know that the model does not support the requested attention.
|
||||
"""
|
||||
dtype = self.config.dtype
|
||||
|
||||
@ -2329,11 +2328,6 @@ class PreTrainedModel(nn.Module, EmbeddingAccessMixin, ModuleUtilsMixin, PushToH
|
||||
|
||||
# With the early check, the parameters are not yet initialized correctly
|
||||
if not is_init_check:
|
||||
if getattr(self, "use_bettertransformer", False):
|
||||
raise ValueError(
|
||||
"Flash Attention 2 and BetterTransformer API are not compatible. Please make sure to disable BetterTransformers by doing model.reverse_bettertransformer()"
|
||||
)
|
||||
|
||||
param_devices = list({param.device for param in self.parameters()})
|
||||
if len(param_devices) == 1 and param_devices[0].type == "cpu":
|
||||
if torch.cuda.is_available():
|
||||
@ -2363,9 +2357,9 @@ class PreTrainedModel(nn.Module, EmbeddingAccessMixin, ModuleUtilsMixin, PushToH
|
||||
Args:
|
||||
is_init_check (`bool`, *optional*):
|
||||
Whether this check is performed early, i.e. at __init__ time, or later when the model and its weights are
|
||||
fully instantiated. This is needed as we also check the devices of the weights, and/or if the model uses
|
||||
BetterTransformer, which are only available later after __init__. This allows to raise proper exceptions early
|
||||
before instantiating the full models if we know that the model does not support the requested attention.
|
||||
fully instantiated. This is needed as we also check the devices of the weights, which are only available
|
||||
later after __init__. This allows to raise proper exceptions early before instantiating the full models
|
||||
if we know that the model does not support the requested attention.
|
||||
"""
|
||||
dtype = self.config.dtype
|
||||
|
||||
@ -2440,9 +2434,9 @@ class PreTrainedModel(nn.Module, EmbeddingAccessMixin, ModuleUtilsMixin, PushToH
|
||||
Args:
|
||||
is_init_check (`bool`, *optional*):
|
||||
Whether this check is performed early, i.e. at __init__ time, or later when the model and its weights are
|
||||
fully instantiated. This is needed as we also check the devices of the weights, and/or if the model uses
|
||||
BetterTransformer, which are only available later after __init__. This allows to raise proper exceptions early
|
||||
before instantiating the full models if we know that the model does not support the requested attention.
|
||||
fully instantiated. This is needed as we also check the devices of the weights, which are only available
|
||||
later after __init__. This allows to raise proper exceptions early before instantiating the full models
|
||||
if we know that the model does not support the requested attention.
|
||||
"""
|
||||
if not self._supports_sdpa:
|
||||
raise ValueError(
|
||||
@ -2461,12 +2455,6 @@ class PreTrainedModel(nn.Module, EmbeddingAccessMixin, ModuleUtilsMixin, PushToH
|
||||
)
|
||||
torch.backends.cuda.enable_flash_sdp(False)
|
||||
|
||||
if not is_init_check:
|
||||
if getattr(self, "use_bettertransformer", False):
|
||||
raise ValueError(
|
||||
"SDPA and BetterTransformer API are not compatible. Please make sure to disable BetterTransformers by doing model.reverse_bettertransformer()"
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
def _flex_attn_can_dispatch(self, is_init_check: bool = False) -> bool:
|
||||
@ -2476,9 +2464,9 @@ class PreTrainedModel(nn.Module, EmbeddingAccessMixin, ModuleUtilsMixin, PushToH
|
||||
Args:
|
||||
is_init_check (`bool`, *optional*):
|
||||
Whether this check is performed early, i.e. at __init__ time, or later when the model and its weights are
|
||||
fully instantiated. This is needed as we also check the devices of the weights, and/or if the model uses
|
||||
BetterTransformer, which are only available later after __init__. This allows to raise proper exceptions early
|
||||
before instantiating the full models if we know that the model does not support the requested attention.
|
||||
fully instantiated. This is needed as we also check the devices of the weights, which are only available
|
||||
later after __init__. This allows to raise proper exceptions early before instantiating the full models
|
||||
if we know that the model does not support the requested attention.
|
||||
"""
|
||||
if not self._supports_flex_attn:
|
||||
raise ValueError(
|
||||
@ -2493,12 +2481,6 @@ class PreTrainedModel(nn.Module, EmbeddingAccessMixin, ModuleUtilsMixin, PushToH
|
||||
"PyTorch Flex Attention requirements in Transformers are not met. Please install torch>=2.5.0."
|
||||
)
|
||||
|
||||
if not is_init_check:
|
||||
if getattr(self, "use_bettertransformer", False):
|
||||
raise ValueError(
|
||||
"FlexAttention and BetterTransformer API are not compatible. Please make sure to disable BetterTransformers by doing model.reverse_bettertransformer()"
|
||||
)
|
||||
|
||||
# If no error raise by this point, we can return `True`
|
||||
return True
|
||||
|
||||
@ -2514,9 +2496,9 @@ class PreTrainedModel(nn.Module, EmbeddingAccessMixin, ModuleUtilsMixin, PushToH
|
||||
The attention implementation to check for existence/validity.
|
||||
is_init_check (`bool`, *optional*):
|
||||
Whether this check is performed early, i.e. at __init__ time, or later when the model and its weights are
|
||||
fully instantiated. This is needed as we also check the devices of the weights, and/or if the model uses
|
||||
BetterTransformer, which are only available later after __init__. This allows to raise proper exceptions early
|
||||
before instantiating the full models if we know that the model does not support the requested attention.
|
||||
fully instantiated. This is needed as we also check the devices of the weights, which are only available
|
||||
later after __init__. This allows to raise proper exceptions early before instantiating the full models
|
||||
if we know that the model does not support the requested attention.
|
||||
|
||||
Returns:
|
||||
`str`: The final attention implementation to use, including potential fallbacks from sdpa to eager, or from
|
||||
@ -5409,56 +5391,6 @@ class PreTrainedModel(nn.Module, EmbeddingAccessMixin, ModuleUtilsMixin, PushToH
|
||||
|
||||
cls._auto_class = auto_class
|
||||
|
||||
def to_bettertransformer(self) -> "PreTrainedModel":
|
||||
"""
|
||||
Converts the model to use [PyTorch's native attention
|
||||
implementation](https://pytorch.org/docs/stable/generated/torch.nn.MultiheadAttention.html), integrated to
|
||||
Transformers through [Optimum library](https://huggingface.co/docs/optimum/bettertransformer/overview). Only a
|
||||
subset of all Transformers models are supported.
|
||||
|
||||
PyTorch's attention fastpath allows to speed up inference through kernel fusions and the use of [nested
|
||||
tensors](https://pytorch.org/docs/stable/nested.html). Detailed benchmarks can be found in [this blog
|
||||
post](https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2).
|
||||
|
||||
Returns:
|
||||
[`PreTrainedModel`]: The model converted to BetterTransformer.
|
||||
"""
|
||||
if not is_optimum_available():
|
||||
raise ImportError("The package `optimum` is required to use Better Transformer.")
|
||||
|
||||
from optimum.version import __version__ as optimum_version
|
||||
|
||||
if version.parse(optimum_version) < version.parse("1.7.0"):
|
||||
raise ImportError(
|
||||
f"Please install optimum>=1.7.0 to use Better Transformer. The version {optimum_version} was found."
|
||||
)
|
||||
|
||||
from optimum.bettertransformer import BetterTransformer
|
||||
|
||||
return BetterTransformer.transform(self)
|
||||
|
||||
def reverse_bettertransformer(self):
|
||||
"""
|
||||
Reverts the transformation from [`~PreTrainedModel.to_bettertransformer`] so that the original modeling is
|
||||
used, for example in order to save the model.
|
||||
|
||||
Returns:
|
||||
[`PreTrainedModel`]: The model converted back to the original modeling.
|
||||
"""
|
||||
if not is_optimum_available():
|
||||
raise ImportError("The package `optimum` is required to use Better Transformer.")
|
||||
|
||||
from optimum.version import __version__ as optimum_version
|
||||
|
||||
if version.parse(optimum_version) < version.parse("1.7.0"):
|
||||
raise ImportError(
|
||||
f"Please install optimum>=1.7.0 to use Better Transformer. The version {optimum_version} was found."
|
||||
)
|
||||
|
||||
from optimum.bettertransformer import BetterTransformer
|
||||
|
||||
return BetterTransformer.reverse(self)
|
||||
|
||||
def warn_if_padding_and_no_attention_mask(self, input_ids, attention_mask):
|
||||
"""
|
||||
Shows a one-time warning if the input_ids appear to contain padding and no attention mask was given.
|
||||
|
||||
@ -1,85 +0,0 @@
|
||||
# Copyright 2023 The HuggingFace Team Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
|
||||
from transformers.testing_utils import (
|
||||
is_torch_available,
|
||||
require_optimum,
|
||||
require_torch,
|
||||
slow,
|
||||
)
|
||||
|
||||
|
||||
if is_torch_available():
|
||||
import torch
|
||||
|
||||
|
||||
@require_torch
|
||||
@require_optimum
|
||||
@slow
|
||||
class BetterTransformerIntegrationTest(unittest.TestCase):
|
||||
# refer to the full test suite in Optimum library:
|
||||
# https://github.com/huggingface/optimum/tree/main/tests/bettertransformer
|
||||
|
||||
def test_transform_and_reverse(self):
|
||||
r"""
|
||||
Classic tests to simply check if the conversion has been successful.
|
||||
"""
|
||||
model_id = "hf-internal-testing/tiny-random-t5"
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
||||
model = AutoModelForSeq2SeqLM.from_pretrained(model_id)
|
||||
|
||||
inp = tokenizer("This is me", return_tensors="pt")
|
||||
|
||||
model = model.to_bettertransformer()
|
||||
|
||||
self.assertTrue(any("BetterTransformer" in mod.__class__.__name__ for _, mod in model.named_modules()))
|
||||
|
||||
output = model.generate(**inp)
|
||||
|
||||
model = model.reverse_bettertransformer()
|
||||
|
||||
self.assertFalse(any("BetterTransformer" in mod.__class__.__name__ for _, mod in model.named_modules()))
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
model.save_pretrained(tmpdirname)
|
||||
|
||||
model_reloaded = AutoModelForSeq2SeqLM.from_pretrained(tmpdirname)
|
||||
|
||||
self.assertFalse(
|
||||
any("BetterTransformer" in mod.__class__.__name__ for _, mod in model_reloaded.named_modules())
|
||||
)
|
||||
|
||||
output_from_pretrained = model_reloaded.generate(**inp)
|
||||
torch.testing.assert_close(output, output_from_pretrained)
|
||||
|
||||
def test_error_save_pretrained(self):
|
||||
r"""
|
||||
The save_pretrained method should raise a ValueError if the model is in BetterTransformer mode.
|
||||
All should be good if the model is reversed.
|
||||
"""
|
||||
model_id = "hf-internal-testing/tiny-random-t5"
|
||||
model = AutoModelForSeq2SeqLM.from_pretrained(model_id)
|
||||
|
||||
model = model.to_bettertransformer()
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
with self.assertRaises(ValueError):
|
||||
model.save_pretrained(tmpdirname)
|
||||
|
||||
model = model.reverse_bettertransformer()
|
||||
model.save_pretrained(tmpdirname)
|
||||
Reference in New Issue
Block a user